diff options
191 files changed, 3069 insertions, 6379 deletions
diff --git a/README.rst b/README.rst index d4095e687..c87b7eccb 100644 --- a/README.rst +++ b/README.rst @@ -41,6 +41,7 @@ Table of Contents src/mailman/docs/WebUIin5 src/mailman/docs/ArchiveUIin5 + Mailman modules --------------- @@ -57,7 +58,7 @@ These documents are generated from the doctest suite. src/mailman/app/docs/* src/mailman/styles/docs/* src/mailman/runners/docs/* - src/mailman/pipeline/docs/* + src/mailman/handlers/docs/* src/mailman/rest/docs/* src/mailman/chains/docs/* src/mailman/rules/docs/* @@ -224,13 +224,14 @@ def index_html(): import errno cwd = os.getcwd() try: - os.makedirs('build/sphinx/html') + try: + os.makedirs('build/sphinx/html') + except OSError as error: + if error.errno != errno.EEXIST: + raise os.chdir('build/sphinx/html') os.symlink('README.html', 'index.html') print 'index.html -> README.html' - except OSError as error: - if error.errno != errno.EEXIST: - raise finally: os.chdir(cwd) diff --git a/src/mailman/Archiver/Archiver.py b/src/mailman/Archiver/Archiver.py deleted file mode 100644 index 1e2af535f..000000000 --- a/src/mailman/Archiver/Archiver.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - # USA. - -"""Mixin class for putting new messages in the right place for archival. - -Public archives are separated from private ones. An external archival -mechanism (eg, pipermail) should be pointed to the right places, to do the -archival. -""" - -import os -import errno -import logging -import mailbox - -from cStringIO import StringIO -from string import Template -from zope.component import getUtility - -from mailman.config import config -from mailman.interfaces.domain import IDomainManager -from mailman.utilities.i18n import make - -log = logging.getLogger('mailman.error') - - - -def makelink(old, new): - try: - os.symlink(old, new) - except OSError, e: - if e.errno <> errno.EEXIST: - raise - -def breaklink(link): - try: - os.unlink(link) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - - - -class Archiver: - # - # Interface to Pipermail. HyperArch.py uses this method to get the - # archive directory for the mailing list - # - def InitVars(self): - # The archive file structure by default is: - # - # archives/ - # private/ - # listname.mbox/ - # listname.mbox - # listname/ - # lots-of-pipermail-stuff - # public/ - # listname.mbox@ -> ../private/listname.mbox - # listname@ -> ../private/listname - # - # IOW, the mbox and pipermail archives are always stored in the - # private archive for the list. This is safe because archives/private - # is always set to o-rx. Public archives have a symlink to get around - # the private directory, pointing directly to the private/listname - # which has o+rx permissions. Private archives do not have the - # symbolic links. - archdir = self.archive_dir(self.fqdn_listname) - omask = os.umask(0) - try: - try: - os.mkdir(archdir+'.mbox', 02775) - except OSError, e: - if e.errno <> errno.EEXIST: - raise - # We also create an empty pipermail archive directory into - # which we'll drop an empty index.html file into. This is so - # that lists that have not yet received a posting have - # /something/ as their index.html, and don't just get a 404. - try: - os.mkdir(archdir, 02775) - except OSError, e: - if e.errno <> errno.EEXIST: - raise - # See if there's an index.html file there already and if not, - # write in the empty archive notice. - indexfile = os.path.join(archdir, 'index.html') - fp = None - try: - fp = open(indexfile) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - omask = os.umask(002) - try: - fp = open(indexfile, 'w') - finally: - os.umask(omask) - fp.write(make('emptyarchive.html', - mailing_list=self, - listname=self.real_name, - listinfo=self.GetScriptURL('listinfo'), - )) - if fp: - fp.close() - finally: - os.umask(omask) - - def ArchiveFileName(self): - """The mbox name where messages are left for archive construction.""" - return os.path.join(self.archive_dir() + '.mbox', - self.fqdn_listname + '.mbox') - - def GetBaseArchiveURL(self): - if self.archive_private: - url = self.GetScriptURL('private') + '/index.html' - else: - domain = getUtility(IDomainManager).get(self.mail_host) - web_host = (self.mail_host if domain is None else domain.url_host) - url = Template(config.PUBLIC_ARCHIVE_URL).safe_substitute( - listname=self.fqdn_listname, - hostname=web_host, - fqdn_listname=self.fqdn_listname, - ) - return url - - def __archive_file(self, afn): - """Open (creating, if necessary) the named archive file.""" - omask = os.umask(002) - try: - return mailbox.mbox(afn, 'a+') - finally: - os.umask(omask) - - # - # old ArchiveMail function, retained under a new name - # for optional archiving to an mbox - # - def __archive_to_mbox(self, post): - """Retain a text copy of the message in an mbox file.""" - try: - afn = self.ArchiveFileName() - mbox = self.__archive_file(afn) - mbox.add(post) - mbox.fp.close() - except IOError, msg: - log.error('Archive file access failure:\n\t%s %s', afn, msg) - raise - - def ExternalArchive(self, ar, txt): - cmd = Template(ar).safe_substitute( - listname=self.fqdn_listname, - hostname=self.mail_host) - extarch = os.popen(cmd, 'w') - extarch.write(txt) - status = extarch.close() - if status: - log.error('external archiver non-zero exit status: %d\n', - (status & 0xff00) >> 8) - - # - # archiving in real time this is called from list.post(msg) - # - def ArchiveMail(self, msg): - """Store postings in mbox and/or pipermail archive, depending.""" - # Fork so archival errors won't disrupt normal list delivery - if config.ARCHIVE_TO_MBOX == -1: - return - # - # We don't need an extra archiver lock here because we know the list - # itself must be locked. - if config.ARCHIVE_TO_MBOX in (1, 2): - self.__archive_to_mbox(msg) - if config.ARCHIVE_TO_MBOX == 1: - # Archive to mbox only. - return - txt = str(msg) - # should we use the internal or external archiver? - private_p = self.archive_private - if config.PUBLIC_EXTERNAL_ARCHIVER and not private_p: - self.ExternalArchive(config.PUBLIC_EXTERNAL_ARCHIVER, txt) - elif config.PRIVATE_EXTERNAL_ARCHIVER and private_p: - self.ExternalArchive(config.PRIVATE_EXTERNAL_ARCHIVER, txt) - else: - # use the internal archiver - f = StringIO(txt) - import HyperArch - h = HyperArch.HyperArchive(self) - h.processUnixMailbox(f) - h.close() - f.close() - - # - # called from MailList.MailList.Save() - # - def CheckHTMLArchiveDir(self): - # We need to make sure that the archive directory has the right perms - # for public vs private. If it doesn't exist, or some weird - # permissions errors prevent us from stating the directory, it's - # pointless to try to fix the perms, so we just return -scott - if config.ARCHIVE_TO_MBOX == -1: - # Archiving is completely disabled, don't require the skeleton. - return - pubdir = os.path.join(config.PUBLIC_ARCHIVE_FILE_DIR, - self.fqdn_listname) - privdir = self.archive_dir() - pubmbox = pubdir + '.mbox' - privmbox = privdir + '.mbox' - if self.archive_private: - breaklink(pubdir) - breaklink(pubmbox) - else: - # BAW: privdir or privmbox could be nonexistant. We'd get an - # OSError, ENOENT which should be caught and reported properly. - makelink(privdir, pubdir) - # Only make this link if the site has enabled public mbox files - if config.PUBLIC_MBOX: - makelink(privmbox, pubmbox) diff --git a/src/mailman/Archiver/HyperArch.py b/src/mailman/Archiver/HyperArch.py deleted file mode 100644 index 017c14342..000000000 --- a/src/mailman/Archiver/HyperArch.py +++ /dev/null @@ -1,1233 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""HyperArch: Pipermail archiving for Mailman - - - The Dragon De Monsyne <dragondm@integral.org> - - TODO: - - Should be able to force all HTML to be regenerated next time the - archive is run, in case a template is changed. - - Run a command to generate tarball of html archives for downloading - (probably in the 'update_dirty_archives' method). -""" - -import os -import re -import sys -import gzip -import time -import errno -import urllib -import logging -import binascii - -from email.Charset import Charset -from email.Errors import HeaderParseError -from email.Header import decode_header, make_header -from flufl.lock import Lock, TimeOutError -from lazr.config import as_boolean -from string import Template -from zope.component import getUtility - -from mailman.Archiver import HyperDatabase -from mailman.Archiver import pipermail -from mailman.config import config -from mailman.core.i18n import _, ctime -from mailman.interfaces.listmanager import IListManager -from mailman.utilities.i18n import find -from mailman.utilities.string import uncanonstr, websafe - - -log = logging.getLogger('mailman.error') -EMPTYSTRING = '' -NL = '\n' - - -# MacOSX has a default stack size that is too small for deeply recursive -# regular expressions. We see this as crashes in the Python test suite when -# running test_re.py and test_sre.py. The fix is to set the stack limit to -# 2048; the general recommendation is to do in the shell before running the -# test suite. But that's inconvenient for a daemon like the runner. -# -# AFAIK, this problem only affects the archiver, so we're adding this work -# around to this file (it'll get imported by the bundled pipermail or by the -# bin/arch script. We also only do this on darwin, a.k.a. MacOSX. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - - -def html_quote(s, langcode=None): - repls = ( ('&', '&'), - ("<", '<'), - (">", '>'), - ('"', '"')) - for thing, repl in repls: - s = s.replace(thing, repl) - return uncanonstr(s, langcode) - - -def url_quote(s): - return urllib.quote(s) - - -def null_to_space(s): - return s.replace('\000', ' ') - - -def sizeof(filename, lang): - try: - size = os.path.getsize(filename) - except OSError, e: - # ENOENT can happen if the .mbox file was moved away or deleted, and - # an explicit mbox file name was given to bin/arch. - if e.errno <> errno.ENOENT: raise - return _('size not available') - if size < 1000: - with _.using(lang.code): - out = _(' %(size)i bytes ') - return out - elif size < 1000000: - return ' %d KB ' % (size / 1000) - # GB?? :-) - return ' %d MB ' % (size / 1000000) - - -html_charset = '<META http-equiv="Content-Type" ' \ - 'content="text/html; charset=%s">' - -def CGIescape(arg, lang=None): - if isinstance(arg, unicode): - s = websafe(arg) - else: - s = websafe(str(arg)) - return uncanonstr(s.replace('"', '"'), lang.code) - -# Parenthesized human name -paren_name_pat = re.compile(r'([(].*[)])') - -# Subject lines preceded with 'Re:' -REpat = re.compile( r"\s*RE\s*(\[\d+\]\s*)?:\s*", re.IGNORECASE) - -# E-mail addresses and URLs in text -emailpat = re.compile(r'([-+,.\w]+@[-+.\w]+)') - -# Argh! This pattern is buggy, and will choke on URLs with GET parameters. -urlpat = re.compile(r'(\w+://[^>)\s]+)') # URLs in text - -# Blank lines -blankpat = re.compile(r'^\s*$') - -# Starting <html> directive -htmlpat = re.compile(r'^\s*<HTML>\s*$', re.IGNORECASE) -# Ending </html> directive -nohtmlpat = re.compile(r'^\s*</HTML>\s*$', re.IGNORECASE) -# Match quoted text -quotedpat = re.compile(r'^([>|:]|>)+') - - - -# Like Utils.maketext() but with caching to improve performance. -# -# _templatefilepathcache is used to associate a (templatefile, lang, listname) -# key with the file system path to a template file. This path is the one that -# the Utils.findtext() function has computed is the one to match the values in -# the key tuple. -# -# _templatecache associate a file system path as key with the text -# returned after processing the contents of that file by Utils.findtext() -# -# We keep two caches to reduce the amount of template text kept in memory, -# since the _templatefilepathcache is a many->one mapping and _templatecache -# is a one->one mapping. Imagine 1000 lists all using the same default -# English template. - -_templatefilepathcache = {} -_templatecache = {} - -def quick_maketext(templatefile, dict=None, lang=None, mlist=None): - if mlist is None: - listname = '' - else: - listname = mlist.fqdn_listname - if lang is None: - if mlist is None: - lang = config.mailman.default_language - else: - lang = mlist.preferred_language - cachekey = (templatefile, lang, listname) - filepath = _templatefilepathcache.get(cachekey) - if filepath: - template = _templatecache.get(filepath) - if filepath is None or template is None: - # Use the basic maketext, with defaults to get the raw template - template, filepath = find(templatefile, mlist=mlist, - language=lang.code) - _templatefilepathcache[cachekey] = filepath - _templatecache[filepath] = template - # Copied from Utils.maketext() - text = template - if dict is not None: - try: - try: - text = Template(template).safe_substitute(**dict) - except UnicodeError: - # Try again after coercing the template to unicode - utemplate = unicode(template, lang.charset, 'replace') - text = Template(utemplate).safe_substitute(**dict) - except (TypeError, ValueError): - # The template is really screwed up - pass - # Make sure the text is in the given character set, or html-ify any bogus - # characters. - return uncanonstr(text, lang.code) - - - -# Note: I'm overriding most, if not all of the pipermail Article class -# here -ddm -# The Article class encapsulates a single posting. The attributes are: -# -# sequence : Sequence number, unique for each article in a set of archives -# subject : Subject -# datestr : The posting date, in human-readable format -# date : The posting date, in purely numeric format -# fromdate : The posting date, in `unixfrom' format -# headers : Any other headers of interest -# author : The author's name (and possibly organization) -# email : The author's e-mail address -# msgid : A unique message ID -# in_reply_to : If !="", this is the msgid of the article being replied to -# references: A (possibly empty) list of msgid's of earlier articles in -# the thread -# body : A list of strings making up the message body - -class Article(pipermail.Article): - __super_init = pipermail.Article.__init__ - __super_set_date = pipermail.Article._set_date - - _last_article_time = time.time() - - def __init__(self, message=None, sequence=0, keepHeaders=[], - lang=config.mailman.default_language, mlist=None): - self.__super_init(message, sequence, keepHeaders) - self.prev = None - self.next = None - # Trim Re: from the subject line - i = 0 - while i != -1: - result = REpat.match(self.subject) - if result: - i = result.end(0) - self.subject = self.subject[i:] - else: - i = -1 - # Useful to keep around - self._lang = lang - self._mlist = mlist - - if as_boolean(config.archiver.pipermail.obscure_email_addresses): - # Avoid i18n side-effects. Note that the language for this - # article (for this list) could be different from the site-wide - # preferred language, so we need to ensure no side-effects will - # occur. Think what happens when executing bin/arch. - with _.using(lang.code): - if self.author == self.email: - self.author = self.email = re.sub('@', _(' at '), - self.email) - else: - self.email = re.sub('@', _(' at '), self.email) - # Snag the content-* headers. RFC 1521 states that their values are - # case insensitive. - ctype = message.get('Content-Type', 'text/plain') - cenc = message.get('Content-Transfer-Encoding', '') - self.ctype = ctype.lower() - self.cenc = cenc.lower() - self.decoded = {} - cset = mlist.preferred_language.charset - cset_out = Charset(cset).output_charset or cset - charset = message.get_content_charset(cset_out) - if charset: - charset = charset.lower().strip() - if charset[0]=='"' and charset[-1]=='"': - charset = charset[1:-1] - if charset[0]=="'" and charset[-1]=="'": - charset = charset[1:-1] - try: - body = message.get_payload(decode=True) - except binascii.Error: - body = None - if body and charset != self._lang.charset: - # decode body - try: - body = unicode(body, charset) - except (UnicodeError, LookupError): - body = None - if body: - self.body = [l + "\n" for l in body.splitlines()] - - self.decode_headers() - - def __getstate__(self): - d = self.__dict__.copy() - # We definitely don't want to pickle the MailList instance, so just - # pickle a reference to it. - if d.has_key('_mlist'): - mlist = d['_mlist'] - del d['_mlist'] - else: - mlist = None - if mlist: - d['__listname'] = self._mlist.fqdn_listname - else: - d['__listname'] = None - # Delete a few other things we don't want in the pickle - for attr in ('prev', 'next', 'body'): - if d.has_key(attr): - del d[attr] - d['body'] = [] - return d - - def __setstate__(self, d): - # For loading older Articles via pickle. All this stuff was added - # when Simone Piunni and Tokio Kikuchi i18n'ified Pipermail. See SF - # patch #594771. - self.__dict__ = d - listname = d.get('__listname') - if listname: - del d['__listname'] - d['_mlist'] = getUtility(IListManager).get(listname) - if not d.has_key('_lang'): - if hasattr(self, '_mlist'): - self._lang = self._mlist.preferred_language - else: - self._lang = config.mailman.default_language - if not d.has_key('cenc'): - self.cenc = None - if not d.has_key('decoded'): - self.decoded = {} - - def setListIfUnset(self, mlist): - if getattr(self, '_mlist', None) is None: - self._mlist = mlist - - def quote(self, buf): - return html_quote(buf, self._lang.code) - - def decode_headers(self): - """MIME-decode headers. - - If the email, subject, or author attributes contain non-ASCII - characters using the encoded-word syntax of RFC 2047, decoded versions - of those attributes are placed in the self.decoded (a dictionary). - - If the list's charset differs from the header charset, an attempt is - made to decode the headers as Unicode. If that fails, they are left - undecoded. - """ - author = self.decode_charset(self.author) - subject = self.decode_charset(self.subject) - if author: - self.decoded['author'] = author - email = self.decode_charset(self.email) - if email: - self.decoded['email'] = email - if subject: - if as_boolean(config.archiver.pipermail.obscure_email_addresses): - with _.using(self._lang.code): - atmark = _(' at ') - subject = re.sub(r'([-+,.\w]+)@([-+.\w]+)', - '\g<1>' + atmark + '\g<2>', subject) - self.decoded['subject'] = subject - self.decoded['stripped'] = self.strip_subject(subject or self.subject) - - def strip_subject(self, subject): - # Strip subject_prefix and Re: for subject sorting - # This part was taken from CookHeaders.py (TK) - prefix = self._mlist.subject_prefix.strip() - if prefix: - prefix_pat = re.escape(prefix) - prefix_pat = '%'.join(prefix_pat.split(r'\%')) - prefix_pat = re.sub(r'%\d*d', r'\s*\d+\s*', prefix_pat) - subject = re.sub(prefix_pat, '', subject) - subject = subject.lstrip() - strip_pat = re.compile('^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I) - stripped = strip_pat.sub('', subject) - return stripped - - def decode_charset(self, field): - # TK: This function was rewritten for unifying to Unicode. - # Convert 'field' into Unicode one line string. - try: - pairs = decode_header(field) - ustr = make_header(pairs).__unicode__() - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # assume list's language - cset = self._mlist.preferred_language.charset - if cset == 'us-ascii': - cset = 'iso-8859-1' # assume this for English list - ustr = unicode(field, cset, 'replace') - return u''.join(ustr.splitlines()) - - def as_html(self): - d = self.__dict__.copy() - # Avoid i18n side-effects - with _.using(self._lang.code): - d["prev"], d["prev_wsubj"] = self._get_prev() - d["next"], d["next_wsubj"] = self._get_next() - - d["email_html"] = self.quote(self.email) - d["title"] = self.quote(self.subject) - d["subject_html"] = self.quote(self.subject) - d["subject_url"] = url_quote(self.subject) - d["in_reply_to_url"] = url_quote(self.in_reply_to) - if as_boolean(config.archiver.pipermail.obscure_email_addresses): - # Point the mailto url back to the list - author = re.sub('@', _(' at '), self.author) - emailurl = self._mlist.posting_address - else: - author = self.author - emailurl = self.email - d["author_html"] = self.quote(author) - d["email_url"] = url_quote(emailurl) - d["datestr_html"] = self.quote(ctime(int(self.date))) - d["body"] = self._get_body() - d['listurl'] = self._mlist.script_url('listinfo') - d['listname'] = self._mlist.real_name - d['encoding'] = '' - charset = self._lang.charset - d["encoding"] = html_charset % charset - - self._add_decoded(d) - return quick_maketext( - 'article.html', d, - lang=self._lang, mlist=self._mlist) - - def _get_prev(self): - """Return the href and subject for the previous message""" - if self.prev: - subject = self._get_subject_enc(self.prev) - prev = ('<LINK REL="Previous" HREF="%s">' - % (url_quote(self.prev.filename))) - prev_wsubj = ('<LI>' + _('Previous message (by thread):') + - ' <A HREF="%s">%s\n</A></li>' - % (url_quote(self.prev.filename), - self.quote(subject))) - else: - prev = prev_wsubj = "" - return prev, prev_wsubj - - def _get_subject_enc(self, art): - """Return the subject of art, decoded if possible. - - If the charset of the current message and art match and the - article's subject is encoded, decode it. - """ - return art.decoded.get('subject', art.subject) - - def _get_next(self): - """Return the href and subject for the previous message""" - if self.next: - subject = self._get_subject_enc(self.next) - next = ('<LINK REL="Next" HREF="%s">' - % (url_quote(self.next.filename))) - next_wsubj = ('<LI>' + _('Next message (by thread):') + - ' <A HREF="%s">%s\n</A></li>' - % (url_quote(self.next.filename), - self.quote(subject))) - else: - next = next_wsubj = "" - return next, next_wsubj - - _rx_quote = re.compile('=([A-F0-9][A-F0-9])') - _rx_softline = re.compile('=[ \t]*$') - - def _get_body(self): - """Return the message body ready for HTML, decoded if necessary""" - try: - body = self.html_body - except AttributeError: - body = self.body - return null_to_space(EMPTYSTRING.join(body)) - - def _add_decoded(self, d): - """Add encoded-word keys to HTML output""" - for src, dst in (('author', 'author_html'), - ('email', 'email_html'), - ('subject', 'subject_html'), - ('subject', 'title')): - if self.decoded.has_key(src): - d[dst] = self.quote(self.decoded[src]) - - def as_text(self): - d = self.__dict__.copy() - # We need to guarantee a valid From_ line, even if there are - # bososities in the headers. - if not d.get('fromdate', '').strip(): - d['fromdate'] = time.ctime(time.time()) - if not d.get('email', '').strip(): - d['email'] = 'bogus@does.not.exist.com' - if not d.get('datestr', '').strip(): - d['datestr'] = time.ctime(time.time()) - # - headers = ['From %(email)s %(fromdate)s', - 'From: %(email)s (%(author)s)', - 'Date: %(datestr)s', - 'Subject: %(subject)s'] - if d['_in_reply_to']: - headers.append('In-Reply-To: %(_in_reply_to)s') - if d['_references']: - headers.append('References: %(_references)s') - if d['_message_id']: - headers.append('Message-ID: %(_message_id)s') - body = EMPTYSTRING.join(self.body) - cset = self._lang.charset - # Coerce the body to Unicode and replace any invalid characters. - if not isinstance(body, unicode): - body = unicode(body, cset, 'replace') - if as_boolean(config.archiver.pipermail.obscure_email_addresses): - with _.using(self._lang.code): - atmark = _(' at ') - body = re.sub(r'([-+,.\w]+)@([-+.\w]+)', - '\g<1>' + atmark + '\g<2>', body) - # Return body to character set of article. - body = body.encode(cset, 'replace') - return NL.join(headers) % d + '\n\n' + body + '\n' - - def _set_date(self, message): - self.__super_set_date(message) - self.fromdate = time.ctime(int(self.date)) - - def loadbody_fromHTML(self,fileobj): - self.body = [] - begin = 0 - while 1: - line = fileobj.readline() - if not line: - break - if not begin: - if line.strip() == '<!--beginarticle-->': - begin = 1 - continue - if line.strip() == '<!--endarticle-->': - break - self.body.append(line) - - def finished_update_article(self): - self.body = [] - try: - del self.html_body - except AttributeError: - pass - - -class HyperArchive(pipermail.T): - __super_init = pipermail.T.__init__ - __super_update_archive = pipermail.T.update_archive - __super_update_dirty_archives = pipermail.T.update_dirty_archives - __super_add_article = pipermail.T.add_article - - # some defaults - DIRMODE = 02775 - FILEMODE = 0660 - - VERBOSE = 0 - DEFAULTINDEX = 'thread' - ARCHIVE_PERIOD = 'month' - - THREADLAZY = 0 - THREADLEVELS = 3 - - ALLOWHTML = 1 # "Lines between <html></html>" handled as is. - SHOWHTML = 0 # Eg, nuke leading whitespace in html manner. - IQUOTES = 1 # Italicize quoted text. - SHOWBR = 0 # Add <br> onto every line - - def __init__(self, maillist): - # can't init the database while other processes are writing to it! - dir = maillist.archive_dir() - db = HyperDatabase.HyperDatabase(dir, maillist) - self.__super_init(dir, reload=1, database=db) - - self.maillist = maillist - self._lock_file = None - self.lang = maillist.preferred_language - self.charset = maillist.preferred_language.charset - - if hasattr(self.maillist,'archive_volume_frequency'): - if self.maillist.archive_volume_frequency == 0: - self.ARCHIVE_PERIOD='year' - elif self.maillist.archive_volume_frequency == 2: - self.ARCHIVE_PERIOD='quarter' - elif self.maillist.archive_volume_frequency == 3: - self.ARCHIVE_PERIOD='week' - elif self.maillist.archive_volume_frequency == 4: - self.ARCHIVE_PERIOD='day' - else: - self.ARCHIVE_PERIOD='month' - - yre = r'(?P<year>[0-9]{4,4})' - mre = r'(?P<month>[01][0-9])' - dre = r'(?P<day>[0123][0-9])' - self._volre = { - 'year': '^' + yre + '$', - 'quarter': '^' + yre + r'q(?P<quarter>[1234])$', - 'month': '^' + yre + r'-(?P<month>[a-zA-Z]+)$', - 'week': r'^Week-of-Mon-' + yre + mre + dre, - 'day': '^' + yre + mre + dre + '$' - } - - def _makeArticle(self, msg, sequence): - return Article(msg, sequence, - lang=self.maillist.preferred_language, - mlist=self.maillist) - - def html_foot(self): - mlist = self.maillist - # Convenience - def quotetime(s): - return html_quote(ctime(s), self.lang.code) - # Avoid i18n side-effects - with _.using(mlist.preferred_language.code): - d = {"lastdate": quotetime(self.lastdate), - "archivedate": quotetime(self.archivedate), - "listinfo": mlist.script_url('listinfo'), - "version": self.version, - } - i = {"thread": _("thread"), - "subject": _("subject"), - "author": _("author"), - "date": _("date") - } - for t in i.keys(): - cap = t[0].upper() + t[1:] - if self.type == cap: - d["%s_ref" % (t)] = "" - else: - d["%s_ref" % (t)] = ('<a href="%s.html#start">[ %s ]</a>' - % (t, i[t])) - return quick_maketext( - 'archidxfoot.html', d, - mlist=mlist) - - def html_head(self): - mlist = self.maillist - # Convenience - def quotetime(s): - return html_quote(ctime(s), self.lang.code) - # Avoid i18n side-effects - with _.using(mlist.preferred_language.code): - d = {"listname": html_quote(mlist.real_name, self.lang.code), - "archtype": self.type, - "archive": self.volNameToDesc(self.archive), - "listinfo": mlist.script_url('listinfo'), - "firstdate": quotetime(self.firstdate), - "lastdate": quotetime(self.lastdate), - "size": self.size, - } - i = {"thread": _("thread"), - "subject": _("subject"), - "author": _("author"), - "date": _("date"), - } - for t in i.keys(): - cap = t[0].upper() + t[1:] - if self.type == cap: - d["%s_ref" % (t)] = "" - d["archtype"] = i[t] - else: - d["%s_ref" % (t)] = ('<a href="%s.html#start">[ %s ]</a>' - % (t, i[t])) - if self.charset: - d["encoding"] = html_charset % self.charset - else: - d["encoding"] = "" - return quick_maketext( - 'archidxhead.html', d, - mlist=mlist) - - def html_TOC(self): - mlist = self.maillist - listname = mlist.fqdn_listname - mbox = os.path.join(mlist.archive_dir()+'.mbox', listname+'.mbox') - d = {"listname": mlist.real_name, - "listinfo": mlist.script_url('listinfo'), - "fullarch": '../%s.mbox/%s.mbox' % (listname, listname), - "size": sizeof(mbox, mlist.preferred_language), - 'meta': '', - } - # Avoid i18n side-effects - with _.using(mlist.preferred_language.code): - if not self.archives: - d["noarchive_msg"] = _( - '<P>Currently, there are no archives. </P>') - d["archive_listing_start"] = "" - d["archive_listing_end"] = "" - d["archive_listing"] = "" - else: - d["noarchive_msg"] = "" - d["archive_listing_start"] = quick_maketext( - 'archliststart.html', - lang=mlist.preferred_language, - mlist=mlist) - d["archive_listing_end"] = quick_maketext( - 'archlistend.html', - mlist=mlist) - - accum = [] - for a in self.archives: - accum.append(self.html_TOC_entry(a)) - d["archive_listing"] = EMPTYSTRING.join(accum) - # The TOC is always in the charset of the list's preferred language - d['meta'] += html_charset % mlist.preferred_language.charset - # The site can disable public access to the mbox file. - if as_boolean(config.archiver.pipermail.public_mbox): - template = 'archtoc.html' - else: - template = 'archtocnombox.html' - return quick_maketext(template, d, mlist=mlist) - - def html_TOC_entry(self, arch): - # Check to see if the archive is gzip'd or not - txtfile = os.path.join(self.maillist.archive_dir(), arch + '.txt') - gzfile = txtfile + '.gz' - # which exists? .txt.gz first, then .txt - if os.path.exists(gzfile): - file = gzfile - url = arch + '.txt.gz' - templ = '<td><A href="%(url)s">[ ' + _('Gzip\'d Text%(sz)s') \ - + ']</a></td>' - elif os.path.exists(txtfile): - file = txtfile - url = arch + '.txt' - templ = '<td><A href="%(url)s">[ ' + _('Text%(sz)s') + ']</a></td>' - else: - # neither found? - file = None - # in Python 1.5.2 we have an easy way to get the size - if file: - textlink = templ % { - 'url': url, - 'sz' : sizeof(file, self.maillist.preferred_language) - } - else: - # there's no archive file at all... hmmm. - textlink = '' - return quick_maketext( - 'archtocentry.html', - {'archive': arch, - 'archivelabel': self.volNameToDesc(arch), - 'textlink': textlink - }, - mlist=self.maillist) - - def GetArchLock(self): - if self._lock_file: - return 1 - self._lock_file = Lock( - os.path.join(config.LOCK_DIR, - self.maillist.fqdn_listname + '-arch.lock')) - try: - self._lock_file.lock(timeout=0.5) - except TimeOutError: - return 0 - return 1 - - def DropArchLock(self): - if self._lock_file: - self._lock_file.unlock(unconditionally=1) - self._lock_file = None - - def processListArch(self): - name = self.maillist.ArchiveFileName() - wname= name+'.working' - ename= name+'.err_unarchived' - try: - os.stat(name) - except (IOError,os.error): - #no archive file, nothin to do -ddm - return - - #see if arch is locked here -ddm - if not self.GetArchLock(): - #another archiver is running, nothing to do. -ddm - return - - #if the working file is still here, the archiver may have - # crashed during archiving. Save it, log an error, and move on. - try: - wf = open(wname) - log.error('Archive working file %s present. ' - 'Check %s for possibly unarchived msgs', - wname, ename) - omask = os.umask(007) - try: - ef = open(ename, 'a+') - finally: - os.umask(omask) - ef.seek(1,2) - if ef.read(1) <> '\n': - ef.write('\n') - ef.write(wf.read()) - ef.close() - wf.close() - os.unlink(wname) - except IOError: - pass - os.rename(name,wname) - archfile = open(wname) - self.processUnixMailbox(archfile) - archfile.close() - os.unlink(wname) - self.DropArchLock() - - def get_filename(self, article): - return '%06i.html' % (article.sequence,) - - def get_archives(self, article): - """Return a list of indexes where the article should be filed. - A string can be returned if the list only contains one entry, - and the empty list is legal.""" - res = self.dateToVolName(float(article.date)) - self.message(_("figuring article archives\n")) - self.message(res + "\n") - return res - - def volNameToDesc(self, volname): - volname = volname.strip() - # Don't make these module global constants since we have to runtime - # translate them anyway. - monthdict = [ - '', - _('January'), _('February'), _('March'), _('April'), - _('May'), _('June'), _('July'), _('August'), - _('September'), _('October'), _('November'), _('December') - ] - for each in self._volre.keys(): - match = re.match(self._volre[each], volname) - # Let ValueErrors percolate up - if match: - year = int(match.group('year')) - if each == 'quarter': - d =["", _("First"), _("Second"), _("Third"), _("Fourth") ] - ord = d[int(match.group('quarter'))] - return _("%(ord)s quarter %(year)i") - elif each == 'month': - monthstr = match.group('month').lower() - for i in range(1, 13): - monthname = time.strftime("%B", (1999,i,1,0,0,0,0,1,0)) - if monthstr.lower() == monthname.lower(): - month = monthdict[i] - return _("%(month)s %(year)i") - raise ValueError, "%s is not a month!" % monthstr - elif each == 'week': - month = monthdict[int(match.group("month"))] - day = int(match.group("day")) - return _("The Week Of Monday %(day)i %(month)s %(year)i") - elif each == 'day': - month = monthdict[int(match.group("month"))] - day = int(match.group("day")) - return _("%(day)i %(month)s %(year)i") - else: - return match.group('year') - raise ValueError, "%s is not a valid volname" % volname - -# The following two methods should be inverses of each other. -ddm - - def dateToVolName(self,date): - datetuple=time.localtime(date) - if self.ARCHIVE_PERIOD=='year': - return time.strftime("%Y",datetuple) - elif self.ARCHIVE_PERIOD=='quarter': - if datetuple[1] in [1,2,3]: - return time.strftime("%Yq1",datetuple) - elif datetuple[1] in [4,5,6]: - return time.strftime("%Yq2",datetuple) - elif datetuple[1] in [7,8,9]: - return time.strftime("%Yq3",datetuple) - else: - return time.strftime("%Yq4",datetuple) - elif self.ARCHIVE_PERIOD == 'day': - return time.strftime("%Y%m%d", datetuple) - elif self.ARCHIVE_PERIOD == 'week': - # Reconstruct "seconds since epoch", and subtract weekday - # multiplied by the number of seconds in a day. - monday = time.mktime(datetuple) - datetuple[6] * 24 * 60 * 60 - # Build a new datetuple from this "seconds since epoch" value - datetuple = time.localtime(monday) - return time.strftime("Week-of-Mon-%Y%m%d", datetuple) - # month. -ddm - else: - return time.strftime("%Y-%B",datetuple) - - - def volNameToDate(self, volname): - volname = volname.strip() - for each in self._volre.keys(): - match = re.match(self._volre[each],volname) - if match: - year = int(match.group('year')) - month = 1 - day = 1 - if each == 'quarter': - q = int(match.group('quarter')) - month = (q * 3) - 2 - elif each == 'month': - monthstr = match.group('month').lower() - m = [] - for i in range(1,13): - m.append( - time.strftime("%B",(1999,i,1,0,0,0,0,1,0)).lower()) - try: - month = m.index(monthstr) + 1 - except ValueError: - pass - elif each == 'week' or each == 'day': - month = int(match.group("month")) - day = int(match.group("day")) - try: - return time.mktime((year,month,1,0,0,0,0,1,-1)) - except OverflowError: - return 0.0 - return 0.0 - - def sortarchives(self): - def sf(a, b): - al = self.volNameToDate(a) - bl = self.volNameToDate(b) - if al > bl: - return 1 - elif al < bl: - return -1 - else: - return 0 - if self.ARCHIVE_PERIOD in ('month','year','quarter'): - self.archives.sort(sf) - else: - self.archives.sort() - self.archives.reverse() - - def message(self, msg): - if self.VERBOSE: - f = sys.stderr - f.write(msg) - if msg[-1:] != '\n': - f.write('\n') - f.flush() - - def open_new_archive(self, archive, archivedir): - index_html = os.path.join(archivedir, 'index.html') - try: - os.unlink(index_html) - except: - pass - os.symlink(self.DEFAULTINDEX+'.html',index_html) - - def write_index_header(self): - self.depth=0 - print self.html_head() - if not self.THREADLAZY and self.type=='Thread': - self.message(_("Computing threaded index\n")) - self.updateThreadedIndex() - - def write_index_footer(self): - for i in range(self.depth): - print '</UL>' - print self.html_foot() - - def write_index_entry(self, article): - subject = self.get_header("subject", article) - author = self.get_header("author", article) - if as_boolean(config.archiver.pipermail.obscure_email_addresses): - try: - author = re.sub('@', _(' at '), author) - except UnicodeError: - # Non-ASCII author contains '@' ... no valid email anyway - pass - subject = CGIescape(subject, self.lang) - author = CGIescape(author, self.lang) - - d = { - 'filename': urllib.quote(article.filename), - 'subject': subject, - 'sequence': article.sequence, - 'author': author - } - print quick_maketext( - 'archidxentry.html', d, - mlist=self.maillist) - - def get_header(self, field, article): - # if we have no decoded header, return the encoded one - result = article.decoded.get(field) - if result is None: - return getattr(article, field) - # otherwise, the decoded one will be Unicode - return result - - def write_threadindex_entry(self, article, depth): - if depth < 0: - self.message('depth<0') - depth = 0 - if depth > self.THREADLEVELS: - depth = self.THREADLEVELS - if depth < self.depth: - for i in range(self.depth-depth): - print '</UL>' - elif depth > self.depth: - for i in range(depth-self.depth): - print '<UL>' - print '<!--%i %s -->' % (depth, article.threadKey) - self.depth = depth - self.write_index_entry(article) - - def write_TOC(self): - self.sortarchives() - omask = os.umask(002) - try: - toc = open(os.path.join(self.basedir, 'index.html'), 'w') - finally: - os.umask(omask) - toc.write(self.html_TOC()) - toc.close() - - def write_article(self, index, article, path): - # called by add_article - omask = os.umask(002) - try: - f = open(path, 'w') - finally: - os.umask(omask) - f.write(article.as_html()) - f.close() - - # Write the text article to the text archive. - path = os.path.join(self.basedir, "%s.txt" % index) - omask = os.umask(002) - try: - f = open(path, 'a+') - finally: - os.umask(omask) - f.write(article.as_text()) - f.close() - - def update_archive(self, archive): - self.__super_update_archive(archive) - # only do this if the gzip module was imported globally, and - # gzip'ing was enabled via Defaults.GZIP_ARCHIVE_TXT_FILES. See - # above. - if gzip: - archz = None - archt = None - txtfile = os.path.join(self.basedir, '%s.txt' % archive) - gzipfile = os.path.join(self.basedir, '%s.txt.gz' % archive) - oldgzip = os.path.join(self.basedir, '%s.old.txt.gz' % archive) - try: - # open the plain text file - archt = open(txtfile) - except IOError: - return - try: - os.rename(gzipfile, oldgzip) - archz = gzip.open(oldgzip) - except (IOError, RuntimeError, os.error): - pass - try: - ou = os.umask(002) - newz = gzip.open(gzipfile, 'w') - finally: - # XXX why is this a finally? - os.umask(ou) - if archz: - newz.write(archz.read()) - archz.close() - os.unlink(oldgzip) - # XXX do we really need all this in a try/except? - try: - newz.write(archt.read()) - newz.close() - archt.close() - except IOError: - pass - os.unlink(txtfile) - - _skip_attrs = ('maillist', '_lock_file', 'charset') - - def getstate(self): - d={} - for each in self.__dict__.keys(): - if not (each in self._skip_attrs - or each.upper() == each): - d[each] = self.__dict__[each] - return d - - # Add <A HREF="..."> tags around URLs and e-mail addresses. - - def __processbody_URLquote(self, lines): - # XXX a lot to do here: - # 1. use lines directly, rather than source and dest - # 2. make it clearer - # 3. make it faster - # TK: Prepare for unicode obscure. - atmark = _(' at ') - if lines and isinstance(lines[0], unicode): - atmark = unicode(atmark, self.lang.charset, 'replace') - source = lines[:] - dest = lines - last_line_was_quoted = 0 - for i in xrange(0, len(source)): - Lorig = L = source[i] - prefix = suffix = "" - if L is None: - continue - # Italicise quoted text - if self.IQUOTES: - quoted = quotedpat.match(L) - if quoted is None: - last_line_was_quoted = 0 - else: - quoted = quoted.end(0) - prefix = CGIescape(L[:quoted], self.lang) + '<i>' - suffix = '</I>' - if self.SHOWHTML: - suffix += '<BR>' - if not last_line_was_quoted: - prefix = '<BR>' + prefix - L = L[quoted:] - last_line_was_quoted = 1 - # Check for an e-mail address - L2 = "" - jr = emailpat.search(L) - kr = urlpat.search(L) - while jr is not None or kr is not None: - if jr == None: - j = -1 - else: - j = jr.start(0) - if kr is None: - k = -1 - else: - k = kr.start(0) - if j != -1 and (j < k or k == -1): - text = jr.group(1) - length = len(text) - if as_boolean( - config.archiver.pipermail.obscure_email_addresses): - text = re.sub('@', atmark, text) - URL = self.maillist.script_url('listinfo') - else: - URL = 'mailto:' + text - pos = j - elif k != -1 and (j > k or j == -1): - text = URL = kr.group(1) - length = len(text) - pos = k - else: # j==k - raise ValueError, "j==k: This can't happen!" - #length = len(text) - #self.message("URL: %s %s %s \n" - # % (CGIescape(L[:pos]), URL, CGIescape(text))) - L2 += '%s<A HREF="%s">%s</A>' % ( - CGIescape(L[:pos], self.lang), - html_quote(URL), CGIescape(text, self.lang)) - L = L[pos+length:] - jr = emailpat.search(L) - kr = urlpat.search(L) - if jr is None and kr is None: - L = CGIescape(L, self.lang) - L = prefix + L2 + L + suffix - source[i] = None - dest[i] = L - - # Perform Hypermail-style processing of <HTML></HTML> directives - # in message bodies. Lines between <HTML> and </HTML> will be written - # out precisely as they are; other lines will be passed to func2 - # for further processing . - - def __processbody_HTML(self, lines): - # XXX need to make this method modify in place - source = lines[:] - dest = lines - l = len(source) - i = 0 - while i < l: - while i < l and htmlpat.match(source[i]) is None: - i = i + 1 - if i < l: - source[i] = None - i = i + 1 - while i < l and nohtmlpat.match(source[i]) is None: - dest[i], source[i] = source[i], None - i = i + 1 - if i < l: - source[i] = None - i = i + 1 - - def format_article(self, article): - # called from add_article - # TBD: Why do the HTML formatting here and keep it in the - # pipermail database? It makes more sense to do the html - # formatting as the article is being written as html and toss - # the data after it has been written to the archive file. - lines = filter(None, article.body) - # Handle <HTML> </HTML> directives - if self.ALLOWHTML: - self.__processbody_HTML(lines) - self.__processbody_URLquote(lines) - if not self.SHOWHTML and lines: - lines.insert(0, '<PRE>') - lines.append('</PRE>') - else: - # Do fancy formatting here - if self.SHOWBR: - lines = map(lambda x:x + "<BR>", lines) - else: - for i in range(0, len(lines)): - s = lines[i] - if s[0:1] in ' \t\n': - lines[i] = '<P>' + s - article.html_body = lines - return article - - def update_article(self, arcdir, article, prev, next): - seq = article.sequence - filename = os.path.join(arcdir, article.filename) - self.message(_('Updating HTML for article %(seq)s')) - try: - f = open(filename) - article.loadbody_fromHTML(f) - f.close() - except IOError, e: - if e.errno <> errno.ENOENT: raise - self.message(_('article file %(filename)s is missing!')) - article.prev = prev - article.next = next - omask = os.umask(002) - try: - f = open(filename, 'w') - finally: - os.umask(omask) - f.write(article.as_html()) - f.close() diff --git a/src/mailman/Archiver/HyperDatabase.py b/src/mailman/Archiver/HyperDatabase.py deleted file mode 100644 index fecb544e8..000000000 --- a/src/mailman/Archiver/HyperDatabase.py +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -# -# site modules -# -import os -import marshal -import time -import errno - -# -# package/project modules -# -import pipermail -from flufl.lock import Lock, NotLockedError - -CACHESIZE = pipermail.CACHESIZE - -try: - import cPickle - pickle = cPickle -except ImportError: - import pickle - -# -# we're using a python dict in place of -# of bsddb.btree database. only defining -# the parts of the interface used by class HyperDatabase -# only one thing can access this at a time. -# -class DumbBTree: - """Stores pickles of Article objects - - This dictionary-like object stores pickles of all the Article - objects. The object itself is stored using marshal. It would be - much simpler, and probably faster, to store the actual objects in - the DumbBTree and pickle it. - - TBD: Also needs a more sensible name, like IteratableDictionary or - SortedDictionary. - """ - - def __init__(self, path): - self.current_index = 0 - self.path = path - self.lockfile = Lock(self.path + ".lock") - self.lock() - self.__dirty = 0 - self.dict = {} - self.sorted = [] - self.load() - - def __repr__(self): - return "DumbBTree(%s)" % self.path - - def __sort(self, dirty=None): - if self.__dirty == 1 or dirty: - self.sorted = self.dict.keys() - self.sorted.sort() - self.__dirty = 0 - - def lock(self): - self.lockfile.lock() - - def unlock(self): - try: - self.lockfile.unlock() - except NotLockedError: - pass - - def __delitem__(self, item): - # if first hasn't been called, we can skip the sort - if self.current_index == 0: - del self.dict[item] - self.__dirty = 1 - return - try: - ci = self.sorted[self.current_index] - except IndexError: - ci = None - if ci == item: - try: - ci = self.sorted[self.current_index + 1] - except IndexError: - ci = None - del self.dict[item] - self.__sort(dirty=1) - if ci is not None: - self.current_index = self.sorted.index(ci) - else: - self.current_index = self.current_index + 1 - - def clear(self): - # bulk clearing much faster than deleting each item, esp. with the - # implementation of __delitem__() above :( - self.dict = {} - - def first(self): - self.__sort() # guarantee that the list is sorted - if not self.sorted: - raise KeyError - else: - key = self.sorted[0] - self.current_index = 1 - return key, self.dict[key] - - def last(self): - if not self.sorted: - raise KeyError - else: - key = self.sorted[-1] - self.current_index = len(self.sorted) - 1 - return key, self.dict[key] - - def next(self): - try: - key = self.sorted[self.current_index] - except IndexError: - raise KeyError - self.current_index = self.current_index + 1 - return key, self.dict[key] - - def has_key(self, key): - return self.dict.has_key(key) - - def set_location(self, loc): - if not self.dict.has_key(loc): - raise KeyError - self.current_index = self.sorted.index(loc) - - def __getitem__(self, item): - return self.dict[item] - - def __setitem__(self, item, val): - # if first hasn't been called, then we don't need to worry - # about sorting again - if self.current_index == 0: - self.dict[item] = val - self.__dirty = 1 - return - try: - current_item = self.sorted[self.current_index] - except IndexError: - current_item = item - self.dict[item] = val - self.__sort(dirty=1) - self.current_index = self.sorted.index(current_item) - - def __len__(self): - return len(self.sorted) - - def load(self): - try: - fp = open(self.path) - try: - self.dict = marshal.load(fp) - finally: - fp.close() - except IOError, e: - if e.errno <> errno.ENOENT: raise - pass - except EOFError: - pass - else: - self.__sort(dirty=1) - - def close(self): - omask = os.umask(007) - try: - fp = open(self.path, 'w') - finally: - os.umask(omask) - fp.write(marshal.dumps(self.dict)) - fp.close() - self.unlock() - - -# this is lifted straight out of pipermail with -# the bsddb.btree replaced with above class. -# didn't use inheritance because of all the -# __internal stuff that needs to be here -scott -# -class HyperDatabase(pipermail.Database): - __super_addArticle = pipermail.Database.addArticle - - def __init__(self, basedir, mlist): - self.__cache = {} - self.__currentOpenArchive = None # The currently open indices - self._mlist = mlist - self.basedir = os.path.expanduser(basedir) - # Recently added articles, indexed only by message ID - self.changed={} - - def firstdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - datekey, msgid = self.dateIndex.first() - date = time.asctime(time.localtime(float(datekey[0]))) - except KeyError: - pass - return date - - def lastdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - datekey, msgid = self.dateIndex.last() - date = time.asctime(time.localtime(float(datekey[0]))) - except KeyError: - pass - return date - - def numArticles(self, archive): - self.__openIndices(archive) - return len(self.dateIndex) - - def addArticle(self, archive, article, subject=None, author=None, - date=None): - self.__openIndices(archive) - self.__super_addArticle(archive, article, subject, author, date) - - def __openIndices(self, archive): - if self.__currentOpenArchive == archive: - return - self.__closeIndices() - arcdir = os.path.join(self.basedir, 'database') - omask = os.umask(0) - try: - try: - os.mkdir(arcdir, 02770) - except OSError, e: - if e.errno <> errno.EEXIST: raise - finally: - os.umask(omask) - for i in ('date', 'author', 'subject', 'article', 'thread'): - t = DumbBTree(os.path.join(arcdir, archive + '-' + i)) - setattr(self, i + 'Index', t) - self.__currentOpenArchive = archive - - def __closeIndices(self): - for i in ('date', 'author', 'subject', 'thread', 'article'): - attr = i + 'Index' - if hasattr(self, attr): - index = getattr(self, attr) - if i == 'article': - if not hasattr(self, 'archive_length'): - self.archive_length = {} - l = len(index) - self.archive_length[self.__currentOpenArchive] = l - index.close() - delattr(self, attr) - self.__currentOpenArchive = None - - def close(self): - self.__closeIndices() - - def hasArticle(self, archive, msgid): - self.__openIndices(archive) - return self.articleIndex.has_key(msgid) - - def setThreadKey(self, archive, key, msgid): - self.__openIndices(archive) - self.threadIndex[key]=msgid - - def getArticle(self, archive, msgid): - self.__openIndices(archive) - if not self.__cache.has_key(msgid): - # get the pickled object out of the DumbBTree - buf = self.articleIndex[msgid] - article = self.__cache[msgid] = pickle.loads(buf) - # For upgrading older archives - article.setListIfUnset(self._mlist) - else: - article = self.__cache[msgid] - return article - - def first(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index + 'Index') - try: - key, msgid = index.first() - return msgid - except KeyError: - return None - - def next(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index + 'Index') - try: - key, msgid = index.next() - return msgid - except KeyError: - return None - - def getOldestArticle(self, archive, subject): - self.__openIndices(archive) - subject = subject.lower() - try: - key, tempid=self.subjectIndex.set_location(subject) - self.subjectIndex.next() - [subject2, date]= key.split('\0') - if subject!=subject2: return None - return tempid - except KeyError: - return None - - def newArchive(self, archive): - pass - - def clearIndex(self, archive, index): - self.__openIndices(archive) - if hasattr(self.threadIndex, 'clear'): - self.threadIndex.clear() - return - finished=0 - try: - key, msgid=self.threadIndex.first() - except KeyError: finished=1 - while not finished: - del self.threadIndex[key] - try: - key, msgid=self.threadIndex.next() - except KeyError: finished=1 diff --git a/src/mailman/Archiver/__init__.py b/src/mailman/Archiver/__init__.py deleted file mode 100644 index be0c61ce0..000000000 --- a/src/mailman/Archiver/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -from Archiver import * diff --git a/src/mailman/Archiver/pipermail.py b/src/mailman/Archiver/pipermail.py deleted file mode 100644 index e11cb7173..000000000 --- a/src/mailman/Archiver/pipermail.py +++ /dev/null @@ -1,872 +0,0 @@ -#! /usr/bin/env python - -import os -import re -import sys -import time -import logging -import mailbox - -import cPickle as pickle - -from cStringIO import StringIO -from email.utils import parseaddr, parsedate_tz, mktime_tz, formatdate -from string import lowercase - -__version__ = '0.11 (Mailman edition)' -VERSION = __version__ -CACHESIZE = 100 # Number of slots in the cache - -from mailman.core import errors -from mailman.core.i18n import _ - -SPACE = ' ' - -log = logging.getLogger('mailman.error') - - - -msgid_pat = re.compile(r'(<.*>)') -def strip_separators(s): - "Remove quotes or parenthesization from a Message-ID string" - if not s: - return "" - if s[0] in '"<([' and s[-1] in '">)]': - s = s[1:-1] - return s - -smallNameParts = ['van', 'von', 'der', 'de'] - -def fixAuthor(author): - "Canonicalize a name into Last, First format" - # If there's a comma, guess that it's already in "Last, First" format - if ',' in author: - return author - L = author.split() - i = len(L) - 1 - if i == 0: - return author # The string's one word--forget it - if author.upper() == author or author.lower() == author: - # Damn, the name is all upper- or lower-case. - while i > 0 and L[i-1].lower() in smallNameParts: - i = i - 1 - else: - # Mixed case; assume that small parts of the last name will be - # in lowercase, and check them against the list. - while i>0 and (L[i-1][0] in lowercase or - L[i-1].lower() in smallNameParts): - i = i - 1 - author = SPACE.join(L[-1:] + L[i:-1]) + ', ' + SPACE.join(L[:i]) - return author - -# Abstract class for databases - -class DatabaseInterface: - def __init__(self): pass - def close(self): pass - def getArticle(self, archive, msgid): pass - def hasArticle(self, archive, msgid): pass - def addArticle(self, archive, article, subject=None, author=None, - date=None): pass - def firstdate(self, archive): pass - def lastdate(self, archive): pass - def first(self, archive, index): pass - def next(self, archive, index): pass - def numArticles(self, archive): pass - def newArchive(self, archive): pass - def setThreadKey(self, archive, key, msgid): pass - def getOldestArticle(self, subject): pass - -class Database(DatabaseInterface): - """Define the basic sorting logic for a database - - Assumes that the database internally uses dateIndex, authorIndex, - etc. - """ - - # TBD Factor out more of the logic shared between BSDDBDatabase - # and HyperDatabase and place it in this class. - - def __init__(self): - # This method need not be called by subclasses that do their - # own initialization. - self.dateIndex = {} - self.authorIndex = {} - self.subjectIndex = {} - self.articleIndex = {} - self.changed = {} - - def addArticle(self, archive, article, subject=None, author=None, - date=None): - # create the keys; always end w/ msgid which will be unique - authorkey = (author or article.author, article.date, - article.msgid) - subjectkey = (subject or article.subject, article.date, - article.msgid) - datekey = date or article.date, article.msgid - - # Add the new article - self.dateIndex[datekey] = article.msgid - self.authorIndex[authorkey] = article.msgid - self.subjectIndex[subjectkey] = article.msgid - - self.store_article(article) - self.changed[archive, article.msgid] = None - - parentID = article.parentID - if parentID is not None and self.articleIndex.has_key(parentID): - parent = self.getArticle(archive, parentID) - myThreadKey = parent.threadKey + article.date + '-' - else: - myThreadKey = article.date + '-' - article.threadKey = myThreadKey - key = myThreadKey, article.msgid - self.setThreadKey(archive, key, article.msgid) - - def store_article(self, article): - """Store article without message body to save space""" - # TBD this is not thread safe! - temp = article.body - temp2 = article.html_body - article.body = [] - del article.html_body - self.articleIndex[article.msgid] = pickle.dumps(article) - article.body = temp - article.html_body = temp2 - - -# The Article class encapsulates a single posting. The attributes -# are: -# -# sequence : Sequence number, unique for each article in a set of archives -# subject : Subject -# datestr : The posting date, in human-readable format -# date : The posting date, in purely numeric format -# headers : Any other headers of interest -# author : The author's name (and possibly organization) -# email : The author's e-mail address -# msgid : A unique message ID -# in_reply_to: If != "", this is the msgid of the article being replied to -# references : A (possibly empty) list of msgid's of earlier articles -# in the thread -# body : A list of strings making up the message body - -class Article: - _last_article_time = time.time() - - def __init__(self, message = None, sequence = 0, keepHeaders = []): - if message is None: - return - self.sequence = sequence - - self.parentID = None - self.threadKey = None - # otherwise the current sequence number is used. - id = strip_separators(message['Message-Id']) - if id == "": - self.msgid = str(self.sequence) - else: self.msgid = id - - if message.has_key('Subject'): - self.subject = str(message['Subject']) - else: - self.subject = _('No subject') - if self.subject == "": self.subject = _('No subject') - - self._set_date(message) - - # Figure out the e-mail address and poster's name. Use the From: - # field first, followed by Reply-To: - self.author, self.email = parseaddr(message.get('From', '')) - e = message['Reply-To'] - if not self.email and e is not None: - ignoreauthor, self.email = parseaddr(e) - self.email = strip_separators(self.email) - self.author = strip_separators(self.author) - - if self.author == "": - self.author = self.email - - # Save the In-Reply-To:, References:, and Message-ID: lines - # - # TBD: The original code does some munging on these fields, which - # shouldn't be necessary, but changing this may break code. For - # safety, I save the original headers on different attributes for use - # in writing the plain text periodic flat files. - self._in_reply_to = message['in-reply-to'] - self._references = message['references'] - self._message_id = message['message-id'] - - i_r_t = message['In-Reply-To'] - if i_r_t is None: - self.in_reply_to = '' - else: - match = msgid_pat.search(i_r_t) - if match is None: self.in_reply_to = '' - else: self.in_reply_to = strip_separators(match.group(1)) - - references = message['References'] - if references is None: - self.references = [] - else: - self.references = map(strip_separators, references.split()) - - # Save any other interesting headers - self.headers = {} - for i in keepHeaders: - if message.has_key(i): - self.headers[i] = message[i] - - # Read the message body - s = StringIO(message.get_payload(decode=True)\ - or message.as_string().split('\n\n',1)[1]) - self.body = s.readlines() - - def _set_date(self, message): - def floatdate(header): - missing = [] - datestr = message.get(header, missing) - if datestr is missing: - return None - date = parsedate_tz(datestr) - try: - return mktime_tz(date) - except (TypeError, ValueError, OverflowError): - return None - date = floatdate('date') - if date is None: - date = floatdate('x-list-received-date') - if date is None: - # What's left to try? - date = self._last_article_time + 1 - self._last_article_time = date - self.date = '%011i' % date - self.datestr = message.get('date') \ - or message.get('x-list-received-date') \ - or formatdate(date) - - def __repr__(self): - return '<Article ID = '+repr(self.msgid)+'>' - - def finished_update_article(self): - pass - -# Pipermail formatter class - -class T: - DIRMODE = 0755 # Mode to give to created directories - FILEMODE = 0644 # Mode to give to created files - INDEX_EXT = ".html" # Extension for indexes - - def __init__(self, basedir = None, reload = 1, database = None): - # If basedir isn't provided, assume the current directory - if basedir is None: - self.basedir = os.getcwd() - else: - basedir = os.path.expanduser(basedir) - self.basedir = basedir - self.database = database - - # If the directory doesn't exist, create it. This code shouldn't get - # run anymore, we create the directory in Archiver.py. It should only - # get used by legacy lists created that are only receiving their first - # message in the HTML archive now -- Marc - try: - os.stat(self.basedir) - except os.error, errdata: - errno, errmsg = errdata - if errno != 2: - raise os.error, errdata - else: - self.message(_('Creating archive directory ') + self.basedir) - omask = os.umask(0) - try: - os.mkdir(self.basedir, self.DIRMODE) - finally: - os.umask(omask) - - # Try to load previously pickled state - try: - if not reload: - raise IOError - f = open(os.path.join(self.basedir, 'pipermail.pck'), 'r') - self.message(_('Reloading pickled archive state')) - d = pickle.load(f) - f.close() - for key, value in d.items(): - setattr(self, key, value) - except (IOError, EOFError): - # No pickled version, so initialize various attributes - self.archives = [] # Archives - self._dirty_archives = [] # Archives that will have to be updated - self.sequence = 0 # Sequence variable used for - # numbering articles - self.update_TOC = 0 # Does the TOC need updating? - # - # make the basedir variable work when passed in as an __init__ arg - # and different from the one in the pickle. Let the one passed in - # as an __init__ arg take precedence if it's stated. This way, an - # archive can be moved from one place to another and still work. - # - if basedir != self.basedir: - self.basedir = basedir - - def close(self): - "Close an archive, save its state, and update any changed archives." - self.update_dirty_archives() - self.update_TOC = 0 - self.write_TOC() - # Save the collective state - self.message(_('Pickling archive state into ') - + os.path.join(self.basedir, 'pipermail.pck')) - self.database.close() - del self.database - - omask = os.umask(007) - try: - f = open(os.path.join(self.basedir, 'pipermail.pck'), 'w') - finally: - os.umask(omask) - pickle.dump(self.getstate(), f) - f.close() - - def getstate(self): - # can override this in subclass - return self.__dict__ - - # - # Private methods - # - # These will be neither overridden nor called by custom archivers. - # - - - # Create a dictionary of various parameters that will be passed - # to the write_index_{header,footer} functions - def __set_parameters(self, archive): - # Determine the earliest and latest date in the archive - firstdate = self.database.firstdate(archive) - lastdate = self.database.lastdate(archive) - - # Get the current time - now = time.asctime(time.localtime(time.time())) - self.firstdate = firstdate - self.lastdate = lastdate - self.archivedate = now - self.size = self.database.numArticles(archive) - self.archive = archive - self.version = __version__ - - # Find the message ID of an article's parent, or return None - # if no parent can be found. - - def __findParent(self, article, children = []): - parentID = None - if article.in_reply_to: - parentID = article.in_reply_to - elif article.references: - # Remove article IDs that aren't in the archive - refs = filter(self.articleIndex.has_key, article.references) - if not refs: - return None - maxdate = self.database.getArticle(self.archive, - refs[0]) - for ref in refs[1:]: - a = self.database.getArticle(self.archive, ref) - if a.date > maxdate.date: - maxdate = a - parentID = maxdate.msgid - else: - # Look for the oldest matching subject - try: - key, tempid = \ - self.subjectIndex.set_location(article.subject) - print key, tempid - self.subjectIndex.next() - [subject, date] = key.split('\0') - print article.subject, subject, date - if subject == article.subject and tempid not in children: - parentID = tempid - except KeyError: - pass - return parentID - - # Update the threaded index completely - def updateThreadedIndex(self): - # Erase the threaded index - self.database.clearIndex(self.archive, 'thread') - - # Loop over all the articles - msgid = self.database.first(self.archive, 'date') - while msgid is not None: - try: - article = self.database.getArticle(self.archive, msgid) - except KeyError: - pass - else: - if article.parentID is None or \ - not self.database.hasArticle(self.archive, - article.parentID): - # then - pass - else: - parent = self.database.getArticle(self.archive, - article.parentID) - article.threadKey = parent.threadKey+article.date+'-' - self.database.setThreadKey(self.archive, - (article.threadKey, article.msgid), - msgid) - msgid = self.database.next(self.archive, 'date') - - # - # Public methods: - # - # These are part of the public interface of the T class, but will - # never be overridden (unless you're trying to do something very new). - - # Update a single archive's indices, whether the archive's been - # dirtied or not. - def update_archive(self, archive): - self.archive = archive - self.message(_("Updating index files for archive [%(archive)s]")) - arcdir = os.path.join(self.basedir, archive) - self.__set_parameters(archive) - - for hdr in ('Date', 'Subject', 'Author'): - self._update_simple_index(hdr, archive, arcdir) - - self._update_thread_index(archive, arcdir) - - def _update_simple_index(self, hdr, archive, arcdir): - self.message(" " + hdr) - self.type = hdr - hdr = hdr.lower() - - self._open_index_file_as_stdout(arcdir, hdr) - self.write_index_header() - count = 0 - # Loop over the index entries - msgid = self.database.first(archive, hdr) - while msgid is not None: - try: - article = self.database.getArticle(self.archive, msgid) - except KeyError: - pass - else: - count = count + 1 - self.write_index_entry(article) - msgid = self.database.next(archive, hdr) - # Finish up this index - self.write_index_footer() - self._restore_stdout() - - def _update_thread_index(self, archive, arcdir): - self.message(_(" Thread")) - self._open_index_file_as_stdout(arcdir, "thread") - self.type = 'Thread' - self.write_index_header() - - # To handle the prev./next in thread pointers, we need to - # track articles 5 at a time. - - # Get the first 5 articles - L = [None] * 5 - i = 2 - msgid = self.database.first(self.archive, 'thread') - - while msgid is not None and i < 5: - L[i] = self.database.getArticle(self.archive, msgid) - i = i + 1 - msgid = self.database.next(self.archive, 'thread') - - while L[2] is not None: - article = L[2] - artkey = None - if article is not None: - artkey = article.threadKey - if artkey is not None: - self.write_threadindex_entry(article, artkey.count('-') - 1) - if self.database.changed.has_key((archive,article.msgid)): - a1 = L[1] - a3 = L[3] - self.update_article(arcdir, article, a1, a3) - if a3 is not None: - self.database.changed[(archive, a3.msgid)] = None - if a1 is not None: - key = archive, a1.msgid - if not self.database.changed.has_key(key): - self.update_article(arcdir, a1, L[0], L[2]) - else: - del self.database.changed[key] - if L[0]: - L[0].finished_update_article() - L = L[1:] # Rotate the list - if msgid is None: - L.append(msgid) - else: - L.append(self.database.getArticle(self.archive, msgid)) - msgid = self.database.next(self.archive, 'thread') - - self.write_index_footer() - self._restore_stdout() - - def _open_index_file_as_stdout(self, arcdir, index_name): - path = os.path.join(arcdir, index_name + self.INDEX_EXT) - omask = os.umask(002) - try: - self.__f = open(path, 'w') - finally: - os.umask(omask) - self.__stdout = sys.stdout - sys.stdout = self.__f - - def _restore_stdout(self): - sys.stdout = self.__stdout - self.__f.close() - del self.__f - del self.__stdout - - # Update only archives that have been marked as "changed". - def update_dirty_archives(self): - for i in self._dirty_archives: - self.update_archive(i) - self._dirty_archives = [] - - # Read a Unix mailbox file from the file object <input>, - # and create a series of Article objects. Each article - # object will then be archived. - - def _makeArticle(self, msg, sequence): - return Article(msg, sequence) - - def processUnixMailbox(self, path, start=None, end=None): - mbox = iter(mailbox.mbox(path)) - if start is None: - start = 0 - counter = 0 - while counter < start: - try: - m = next(mbox) - except errors.DiscardMessage: - continue - if m is None: - return - counter += 1 - while True: - try: - m = next(mbox) - except StopIteration: - break - except errors.DiscardMessage: - continue - except Exception: - log.error('uncaught archiver exception') - raise - if m == '': - # It was an unparseable message - continue - msgid = m.get('message-id', 'n/a') - self.message(_('#%(counter)05d %(msgid)s')) - a = self._makeArticle(m, self.sequence) - self.sequence += 1 - self.add_article(a) - if end is not None and counter >= end: - break - counter += 1 - - def new_archive(self, archive, archivedir): - self.archives.append(archive) - self.update_TOC = 1 - self.database.newArchive(archive) - # If the archive directory doesn't exist, create it - try: - os.stat(archivedir) - except os.error, errdata: - errno, errmsg = errdata - if errno == 2: - omask = os.umask(0) - try: - os.mkdir(archivedir, self.DIRMODE) - finally: - os.umask(omask) - else: - raise os.error, errdata - self.open_new_archive(archive, archivedir) - - def add_article(self, article): - archives = self.get_archives(article) - if not archives: - return - if type(archives) == type(''): - archives = [archives] - - article.filename = filename = self.get_filename(article) - temp = self.format_article(article) - for arch in archives: - self.archive = arch # why do this??? - archivedir = os.path.join(self.basedir, arch) - if arch not in self.archives: - self.new_archive(arch, archivedir) - - # Write the HTML-ized article - self.write_article(arch, temp, os.path.join(archivedir, - filename)) - - if article.decoded.has_key('author'): - author = fixAuthor(article.decoded['author']) - else: - author = fixAuthor(article.author) - if article.decoded.has_key('stripped'): - subject = article.decoded['stripped'].lower() - else: - subject = article.subject.lower() - - article.parentID = parentID = self.get_parent_info(arch, article) - if parentID: - parent = self.database.getArticle(arch, parentID) - article.threadKey = parent.threadKey + article.date + '-' - else: - article.threadKey = article.date + '-' - key = article.threadKey, article.msgid - - self.database.setThreadKey(arch, key, article.msgid) - self.database.addArticle(arch, temp, author=author, - subject=subject) - - if arch not in self._dirty_archives: - self._dirty_archives.append(arch) - - def get_parent_info(self, archive, article): - parentID = None - if article.in_reply_to: - parentID = article.in_reply_to - elif article.references: - refs = self._remove_external_references(article.references) - if refs: - maxdate = self.database.getArticle(archive, refs[0]) - for ref in refs[1:]: - a = self.database.getArticle(archive, ref) - if a.date > maxdate.date: - maxdate = a - parentID = maxdate.msgid - else: - # Get the oldest article with a matching subject, and - # assume this is a follow-up to that article - parentID = self.database.getOldestArticle(archive, - article.subject) - - if parentID and not self.database.hasArticle(archive, parentID): - parentID = None - return parentID - - def write_article(self, index, article, path): - omask = os.umask(002) - try: - f = open(path, 'w') - finally: - os.umask(omask) - temp_stdout, sys.stdout = sys.stdout, f - self.write_article_header(article) - sys.stdout.writelines(article.body) - self.write_article_footer(article) - sys.stdout = temp_stdout - f.close() - - def _remove_external_references(self, refs): - keep = [] - for ref in refs: - if self.database.hasArticle(self.archive, ref): - keep.append(ref) - return keep - - # Abstract methods: these will need to be overridden by subclasses - # before anything useful can be done. - - def get_filename(self, article): - pass - def get_archives(self, article): - """Return a list of indexes where the article should be filed. - A string can be returned if the list only contains one entry, - and the empty list is legal.""" - pass - def format_article(self, article): - pass - def write_index_header(self): - pass - def write_index_footer(self): - pass - def write_index_entry(self, article): - pass - def write_threadindex_entry(self, article, depth): - pass - def write_article_header(self, article): - pass - def write_article_footer(self, article): - pass - def write_article_entry(self, article): - pass - def update_article(self, archivedir, article, prev, next): - pass - def write_TOC(self): - pass - def open_new_archive(self, archive, dir): - pass - def message(self, msg): - pass - - -class BSDDBdatabase(Database): - __super_addArticle = Database.addArticle - - def __init__(self, basedir): - self.__cachekeys = [] - self.__cachedict = {} - self.__currentOpenArchive = None # The currently open indices - self.basedir = os.path.expanduser(basedir) - self.changed = {} # Recently added articles, indexed only by - # message ID - - def firstdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - date, msgid = self.dateIndex.first() - date = time.asctime(time.localtime(float(date))) - except KeyError: - pass - return date - - def lastdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - date, msgid = self.dateIndex.last() - date = time.asctime(time.localtime(float(date))) - except KeyError: - pass - return date - - def numArticles(self, archive): - self.__openIndices(archive) - return len(self.dateIndex) - - def addArticle(self, archive, article, subject=None, author=None, - date=None): - self.__openIndices(archive) - self.__super_addArticle(archive, article, subject, author, date) - - # Open the BSDDB files that are being used as indices - # (dateIndex, authorIndex, subjectIndex, articleIndex) - def __openIndices(self, archive): - if self.__currentOpenArchive == archive: - return - - import bsddb - self.__closeIndices() - arcdir = os.path.join(self.basedir, 'database') - omask = os.umask(0) - try: - try: - os.mkdir(arcdir, 02775) - except OSError: - # BAW: Hmm... - pass - finally: - os.umask(omask) - for hdr in ('date', 'author', 'subject', 'article', 'thread'): - path = os.path.join(arcdir, archive + '-' + hdr) - t = bsddb.btopen(path, 'c') - setattr(self, hdr + 'Index', t) - self.__currentOpenArchive = archive - - # Close the BSDDB files that are being used as indices (if they're - # open--this is safe to call if they're already closed) - def __closeIndices(self): - if self.__currentOpenArchive is not None: - pass - for hdr in ('date', 'author', 'subject', 'thread', 'article'): - attr = hdr + 'Index' - if hasattr(self, attr): - index = getattr(self, attr) - if hdr == 'article': - if not hasattr(self, 'archive_length'): - self.archive_length = {} - self.archive_length[self.__currentOpenArchive] = len(index) - index.close() - delattr(self,attr) - self.__currentOpenArchive = None - - def close(self): - self.__closeIndices() - def hasArticle(self, archive, msgid): - self.__openIndices(archive) - return self.articleIndex.has_key(msgid) - def setThreadKey(self, archive, key, msgid): - self.__openIndices(archive) - self.threadIndex[key] = msgid - def getArticle(self, archive, msgid): - self.__openIndices(archive) - if self.__cachedict.has_key(msgid): - self.__cachekeys.remove(msgid) - self.__cachekeys.append(msgid) - return self.__cachedict[msgid] - if len(self.__cachekeys) == CACHESIZE: - delkey, self.__cachekeys = (self.__cachekeys[0], - self.__cachekeys[1:]) - del self.__cachedict[delkey] - s = self.articleIndex[msgid] - article = pickle.loads(s) - self.__cachekeys.append(msgid) - self.__cachedict[msgid] = article - return article - - def first(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index+'Index') - try: - key, msgid = index.first() - return msgid - except KeyError: - return None - def next(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index+'Index') - try: - key, msgid = index.next() - except KeyError: - return None - else: - return msgid - - def getOldestArticle(self, archive, subject): - self.__openIndices(archive) - subject = subject.lower() - try: - key, tempid = self.subjectIndex.set_location(subject) - self.subjectIndex.next() - [subject2, date] = key.split('\0') - if subject != subject2: - return None - return tempid - except KeyError: # XXX what line raises the KeyError? - return None - - def newArchive(self, archive): - pass - - def clearIndex(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index+'Index') - finished = 0 - try: - key, msgid = self.threadIndex.first() - except KeyError: - finished = 1 - while not finished: - del self.threadIndex[key] - try: - key, msgid = self.threadIndex.next() - except KeyError: - finished = 1 - - diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py index a9bed97ac..34d90ac82 100644 --- a/src/mailman/app/bounces.py +++ b/src/mailman/app/bounces.py @@ -59,15 +59,16 @@ DOT = '.' -def bounce_message(mlist, msg, e=None): +def bounce_message(mlist, msg, error=None): """Bounce the message back to the original author. :param mlist: The mailing list that the message was posted to. :type mlist: `IMailingList` :param msg: The original message. :type msg: `email.message.Message` - :param e: Optional exception causing the bounce. - :type e: Exception + :param error: Optional exception causing the bounce. The exception + instance must have a `.message` attribute. + :type error: Exception """ # Bounce a message back to the sender, with an error message if provided # in the exception argument. @@ -77,10 +78,10 @@ def bounce_message(mlist, msg, e=None): return subject = msg.get('subject', _('(no subject)')) subject = oneline(subject, mlist.preferred_language.charset) - if e is None: + if error is None: notice = _('[No bounce details are available]') else: - notice = _(e.notice) + notice = _(error.message) # Currently we always craft bounces as MIME messages. bmsg = UserNotification(msg.sender, mlist.owner_address, subject, lang=mlist.preferred_language) @@ -215,7 +216,7 @@ def send_probe(member, msg): ) # Calculate the Subject header, in the member's preferred language. with _.using(member.preferred_language.code): - subject = _('$mlist.real_name mailing list probe message') + subject = _('$mlist.display_name mailing list probe message') # Craft the probe message. This will be a multipart where the first part # is the probe text and the second part is the message that caused this # probe to be sent. @@ -225,7 +226,9 @@ def send_probe(member, msg): notice = MIMEText(text, _charset=mlist.preferred_language.charset) probe.attach(notice) probe.attach(MIMEMessage(msg)) - probe.send(mlist, envsender=probe_sender, verp=False, probe_token=token) + # Probes should not have the Precedence: bulk header. + probe.send(mlist, envsender=probe_sender, verp=False, probe_token=token, + add_precedence=False) return token diff --git a/src/mailman/app/docs/bounces.rst b/src/mailman/app/docs/bounces.rst index f825064e3..5510f2207 100644 --- a/src/mailman/app/docs/bounces.rst +++ b/src/mailman/app/docs/bounces.rst @@ -12,12 +12,12 @@ Mailman can bounce messages back to the original sender. This is essentially equivalent to rejecting the message with notification. Mailing lists can bounce a message with an optional error message. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('text@example.com') Any message can be bounced. >>> msg = message_from_string("""\ - ... To: _xtest@example.com + ... To: text@example.com ... From: aperson@example.com ... Subject: Something important ... @@ -36,7 +36,7 @@ to the original message author. 1 >>> print items[0].msg.as_string() Subject: Something important - From: _xtest-owner@example.com + From: text-owner@example.com To: aperson@example.com MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="..." @@ -54,7 +54,7 @@ to the original message author. Content-Type: message/rfc822 MIME-Version: 1.0 <BLANKLINE> - To: _xtest@example.com + To: text@example.com From: aperson@example.com Subject: Something important <BLANKLINE> @@ -74,7 +74,7 @@ passed in as an instance of a ``RejectMessage`` exception. 1 >>> print items[0].msg.as_string() Subject: Something important - From: _xtest-owner@example.com + From: text-owner@example.com To: aperson@example.com MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="..." @@ -92,7 +92,7 @@ passed in as an instance of a ``RejectMessage`` exception. Content-Type: message/rfc822 MIME-Version: 1.0 <BLANKLINE> - To: _xtest@example.com + To: text@example.com From: aperson@example.com Subject: Something important <BLANKLINE> diff --git a/src/mailman/app/docs/lifecycle.rst b/src/mailman/app/docs/lifecycle.rst index d8356db74..08a25ccff 100644 --- a/src/mailman/app/docs/lifecycle.rst +++ b/src/mailman/app/docs/lifecycle.rst @@ -119,13 +119,13 @@ the system, they won't be created again. >>> user_b = user_manager.get_user('bperson@example.com') >>> user_c = user_manager.get_user('cperson@example.com') >>> user_d = user_manager.get_user('dperson@example.com') - >>> user_a.real_name = 'Anne Person' - >>> user_b.real_name = 'Bart Person' - >>> user_c.real_name = 'Caty Person' - >>> user_d.real_name = 'Dirk Person' + >>> user_a.display_name = 'Anne Person' + >>> user_b.display_name = 'Bart Person' + >>> user_c.display_name = 'Caty Person' + >>> user_d.display_name = 'Dirk Person' >>> mlist_3 = create_list('test_3@example.com', owners) - >>> dump_list(user.real_name for user in mlist_3.owners.users) + >>> dump_list(user.display_name for user in mlist_3.owners.users) Anne Person Bart Person Caty Person @@ -140,7 +140,7 @@ artifacts. :: >>> from mailman.app.lifecycle import remove_list - >>> remove_list(mlist_2.fqdn_listname, mlist_2, True) + >>> remove_list(mlist_2.fqdn_listname, mlist_2) >>> from mailman.interfaces.listmanager import IListManager >>> from zope.component import getUtility diff --git a/src/mailman/app/docs/message.rst b/src/mailman/app/docs/message.rst index 3e3293196..3c3fd8ea8 100644 --- a/src/mailman/app/docs/message.rst +++ b/src/mailman/app/docs/message.rst @@ -2,7 +2,7 @@ Messages ======== -Mailman has its own Message classes, derived from the standard +Mailman has its own `Message` classes, derived from the standard ``email.message.Message`` class, but providing additional useful methods. @@ -13,7 +13,7 @@ When Mailman needs to send a message to a user, it creates a ``UserNotification`` instance, and then calls the ``.send()`` method on this object. This method requires a mailing list instance. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') The ``UserNotification`` constructor takes the recipient address, the sender address, an optional subject, optional body text, and optional language. @@ -21,28 +21,69 @@ address, an optional subject, optional body text, and optional language. >>> from mailman.email.message import UserNotification >>> msg = UserNotification( ... 'aperson@example.com', - ... '_xtest@example.com', + ... 'test@example.com', ... 'Something you need to know', ... 'I needed to tell you this.') >>> msg.send(mlist) The message will end up in the `virgin` queue. - >>> switchboard = config.switchboards['virgin'] - >>> len(switchboard.files) + >>> from mailman.testing.helpers import get_queue_messages + >>> messages = get_queue_messages('virgin') + >>> len(messages) 1 - >>> filebase = switchboard.files[0] - >>> qmsg, qmsgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print qmsg.as_string() + >>> print messages[0].msg.as_string() MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: Something you need to know - From: _xtest@example.com + From: test@example.com To: aperson@example.com Message-ID: ... Date: ... Precedence: bulk <BLANKLINE> I needed to tell you this. + +The message above got a `Precedence: bulk` header added by default. If the +message we're sending already has a `Precedence:` header, it shouldn't be +changed. + + >>> del msg['precedence'] + >>> msg['Precedence'] = 'list' + >>> msg.send(mlist) + +Again, the message will end up in the `virgin` queue but with the original +`Precedence:` header. + + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> print messages[0].msg['precedence'] + list + +Sometimes we want to send the message without a `Precedence:` header such as +when we send a probe message. + + >>> del msg['precedence'] + >>> msg.send(mlist, add_precedence=False) + +Again, the message will end up in the `virgin` queue but without the +`Precedence:` header. + + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> print messages[0].msg['precedence'] + None + +However, if the message already has a `Precedence:` header, setting the +`precedence=False` argument will have no effect. + + >>> msg['Precedence'] = 'junk' + >>> msg.send(mlist, add_precedence=False) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> print messages[0].msg['precedence'] + junk diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py index 6826d68f1..5082034bc 100644 --- a/src/mailman/app/lifecycle.py +++ b/src/mailman/app/lifecycle.py @@ -89,7 +89,7 @@ def create_list(fqdn_listname, owners=None): -def remove_list(fqdn_listname, mailing_list=None, archives=True): +def remove_list(fqdn_listname, mailing_list=None): """Remove the list and all associated artifacts and subscriptions.""" removeables = [] # mailing_list will be None when only residual archives are being removed. @@ -108,15 +108,6 @@ def remove_list(fqdn_listname, mailing_list=None, archives=True): fn_listname = filename.split('.')[0] if fn_listname == fqdn_listname: removeables.append(os.path.join(config.LOCK_DIR, filename)) - if archives: - private_dir = config.PRIVATE_ARCHIVE_FILE_DIR - public_dir = config.PUBLIC_ARCHIVE_FILE_DIR - removeables.extend([ - os.path.join(private_dir, fqdn_listname), - os.path.join(private_dir, fqdn_listname + '.mbox'), - os.path.join(public_dir, fqdn_listname), - os.path.join(public_dir, fqdn_listname + '.mbox'), - ]) # Now that we know what files and directories to delete, delete them. for target in removeables: if not os.path.exists(target): diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index 1d075afcd..e31a1695c 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -43,7 +43,7 @@ from mailman.utilities.i18n import make -def add_member(mlist, email, realname, password, delivery_mode, language, +def add_member(mlist, email, display_name, password, delivery_mode, language, role=MemberRole.member): """Add a member right now. @@ -54,8 +54,8 @@ def add_member(mlist, email, realname, password, delivery_mode, language, :type mlist: `IMailingList` :param email: The email address to subscribe. :type email: str - :param realname: The subscriber's full name. - :type realname: str + :param display_name: The subscriber's full name. + :type display_name: str :param password: The subscriber's plain text password. :type password: str :param delivery_mode: The delivery mode the subscriber has chosen. @@ -86,14 +86,15 @@ def add_member(mlist, email, realname, password, delivery_mode, language, if address is None: # Nope, we don't even know about this address, so create both the # user and address now. - user = user_manager.create_user(email, realname) + user = user_manager.create_user(email, display_name) # Do it this way so we don't have to flush the previous change. address = list(user.addresses)[0] else: # The address object exists, but it's not linked to a user. # Create the user and link it now. user = user_manager.create_user() - user.real_name = (realname if realname else address.real_name) + user.display_name = ( + display_name if display_name else address.display_name) user.link(address) # Encrypt the password using the currently selected scheme. The # scheme is recorded in the hashed password string. @@ -148,12 +149,12 @@ def delete_member(mlist, email, admin_notif=None, userack=None): # ...and to the administrator. if admin_notif: user = getUtility(IUserManager).get_user(email) - realname = user.real_name - subject = _('$mlist.real_name unsubscription notification') + display_name = user.display_name + subject = _('$mlist.display_name unsubscription notification') text = make('adminunsubscribeack.txt', mailing_list=mlist, - listname=mlist.real_name, - member=formataddr((realname, email)), + listname=mlist.display_name, + member=formataddr((display_name, email)), ) msg = OwnerNotification(mlist, subject, text, roster=mlist.administrators) diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index 01dc4a232..2e2711809 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -162,7 +162,7 @@ def handle_message(mlist, id, action, # Get a copy of the original message from the message store. msg = message_store.get_message_by_id(message_id) # It's possible the forwarding address list is a comma separated list - # of realname/address pairs. + # of display_name/address pairs. addresses = [addr[1] for addr in getaddresses(forward)] language = mlist.preferred_language if len(addresses) == 1: @@ -197,10 +197,10 @@ def handle_message(mlist, id, action, -def hold_subscription(mlist, address, realname, password, mode, language): +def hold_subscription(mlist, address, display_name, password, mode, language): data = dict(when=now().isoformat(), address=address, - realname=realname, + display_name=display_name, password=password, delivery_mode=str(mode), language=language) @@ -213,7 +213,7 @@ def hold_subscription(mlist, address, realname, password, mode, language): # Possibly notify the administrator in default list language if mlist.admin_immed_notify: subject = _( - 'New subscription request to list $mlist.real_name from $address') + 'New subscription request to $mlist.display_name from $address') text = make('subauth.txt', mailing_list=mlist, username=address, @@ -249,11 +249,11 @@ def handle_subscription(mlist, id, action, comment=None): enum_value = data['delivery_mode'].split('.')[-1] delivery_mode = DeliveryMode(enum_value) address = data['address'] - realname = data['realname'] + display_name = data['display_name'] language = getUtility(ILanguageManager)[data['language']] password = data['password'] try: - add_member(mlist, address, realname, password, + add_member(mlist, address, display_name, password, delivery_mode, language) except AlreadySubscribedError: # The address got subscribed in some other way after the original @@ -264,9 +264,9 @@ def handle_subscription(mlist, id, action, comment=None): send_welcome_message(mlist, address, language, delivery_mode) if mlist.admin_notify_mchanges: send_admin_subscription_notice( - mlist, address, realname, language) + mlist, address, display_name, language) slog.info('%s: new %s, %s %s', mlist.fqdn_listname, - delivery_mode, formataddr((realname, address)), + delivery_mode, formataddr((display_name, address)), 'via admin approval') else: raise AssertionError('Unexpected action: {0}'.format(action)) @@ -285,7 +285,7 @@ def hold_unsubscription(mlist, address): # Possibly notify the administrator of the hold if mlist.admin_immed_notify: subject = _( - 'New unsubscription request from $mlist.real_name by $address') + 'New unsubscription request from $mlist.display_name by $address') text = make('unsubauth.txt', mailing_list=mlist, address=address, @@ -335,7 +335,7 @@ def _refuse(mlist, request, recip, comment, origmsg=None, lang=None): # As this message is going to the requester, try to set the language to # his/her language choice, if they are a member. Otherwise use the list's # preferred language. - realname = mlist.real_name + display_name = mlist.display_name if lang is None: member = mlist.members.get_member(recip) lang = (mlist.preferred_language @@ -357,7 +357,7 @@ def _refuse(mlist, request, recip, comment, origmsg=None, lang=None): '---------- ' + _('Original Message') + ' ----------', str(origmsg) ]) - subject = _('Request to mailing list "$realname" rejected') + subject = _('Request to mailing list "$display_name" rejected') msg = UserNotification(recip, mlist.bounces_address, subject, text, lang) msg.send(mlist) diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 5604d5f05..bc5b326ac 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -17,7 +17,7 @@ """Sending notifications.""" -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -80,12 +80,12 @@ def send_welcome_message(mlist, address, language, delivery_mode, text=''): # Find the IMember object which is subscribed to the mailing list, because # from there, we can get the member's options url. member = mlist.members.get_member(address) - user_name = member.user.real_name + user_name = member.user.display_name options_url = member.options_url # Get the text from the template. text = expand(welcome, dict( fqdn_listname=mlist.fqdn_listname, - list_name=mlist.real_name, + list_name=mlist.display_name, listinfo_uri=mlist.script_url('listinfo'), list_requests=mlist.request_address, user_name=user_name, @@ -99,7 +99,7 @@ def send_welcome_message(mlist, address, language, delivery_mode, text=''): msg = UserNotification( formataddr((user_name, address)), mlist.request_address, - _('Welcome to the "$mlist.real_name" mailing list${digmode}'), + _('Welcome to the "$mlist.display_name" mailing list${digmode}'), text, language) msg['X-No-Archive'] = 'yes' msg.send(mlist, verp=as_boolean(config.mta.verp_personalized_deliveries)) @@ -125,31 +125,32 @@ def send_goodbye_message(mlist, address, language): goodbye = '' msg = UserNotification( address, mlist.bounces_address, - _('You have been unsubscribed from the $mlist.real_name mailing list'), + _('You have been unsubscribed from the $mlist.display_name ' + 'mailing list'), goodbye, language) msg.send(mlist, verp=as_boolean(config.mta.verp_personalized_deliveries)) -def send_admin_subscription_notice(mlist, address, full_name, language): +def send_admin_subscription_notice(mlist, address, display_name, language): """Send the list administrators a subscription notice. - :param mlist: the mailing list + :param mlist: The mailing list. :type mlist: IMailingList - :param address: the address being subscribed + :param address: The address being subscribed. :type address: string - :param full_name: the name of the subscriber - :type full_name: string - :param language: the language of the address's realname + :param display_name: The name of the subscriber. + :type display_name: string + :param language: The language of the address's display name. :type language: string """ with _.using(mlist.preferred_language.code): - subject = _('$mlist.real_name subscription notification') - full_name = full_name.encode(language.charset, 'replace') + subject = _('$mlist.display_name subscription notification') + display_name = display_name.encode(language.charset, 'replace') text = make('adminsubscribeack.txt', mailing_list=mlist, - listname=mlist.real_name, - member=formataddr((full_name, address)), + listname=mlist.display_name, + member=formataddr((display_name, address)), ) msg = OwnerNotification(mlist, subject, text, roster=mlist.administrators) msg.send(mlist) diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index e439b3c54..030a504f7 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -17,7 +17,7 @@ """Implementation of the IUserRegistrar interface.""" -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,6 @@ __all__ = [ import logging -from pkg_resources import resource_string from zope.component import getUtility from zope.interface import implements @@ -59,7 +58,7 @@ class Registrar: implements(IRegistrar) - def register(self, mlist, email, real_name=None, delivery_mode=None): + def register(self, mlist, email, display_name=None, delivery_mode=None): """See `IUserRegistrar`.""" if delivery_mode is None: delivery_mode = DeliveryMode.regular @@ -70,7 +69,7 @@ class Registrar: pendable = PendableRegistration( type=PendableRegistration.PEND_KEY, email=email, - real_name=real_name, + display_name=display_name, delivery_mode=delivery_mode.name) pendable['list_name'] = mlist.fqdn_listname token = getUtility(IPendings).add(pendable) @@ -104,7 +103,7 @@ class Registrar: return False missing = object() email = pendable.get('email', missing) - real_name = pendable.get('real_name', missing) + display_name = pendable.get('display_name', missing) list_name = pendable.get('list_name', missing) pended_delivery_mode = pendable.get('delivery_mode', 'regular') try: @@ -133,7 +132,7 @@ class Registrar: # and link the two together if address is None: assert user is None, 'How did we get a user but not an address?' - user = user_manager.create_user(email, real_name) + user = user_manager.create_user(email, display_name) # Because the database changes haven't been flushed, we can't use # IUserManager.get_address() to find the IAddress just created # under the hood. Instead, iterate through the IUser's addresses, @@ -145,7 +144,7 @@ class Registrar: raise AssertionError('Could not find expected IAddress') elif user is None: user = user_manager.create_user() - user.real_name = real_name + user.display_name = display_name user.link(address) else: # The IAddress and linked IUser already exist, so all we need to diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 931d7f4c7..60f8cdebe 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -145,7 +145,7 @@ class SubscriptionService: yield member def join(self, fqdn_listname, subscriber, - real_name=None, + display_name=None, delivery_mode=DeliveryMode.regular, role=MemberRole.member): """See `ISubscriptionService`.""" @@ -158,8 +158,8 @@ class SubscriptionService: # it's a valid email address, and let InvalidEmailAddressError # propagate up. getUtility(IEmailValidator).validate(subscriber) - if real_name is None: - real_name, at, domain = subscriber.partition('@') + if display_name is None: + display_name, at, domain = subscriber.partition('@') # Because we want to keep the REST API simple, there is no # password or language given to us. We'll use the system's # default language for the user's default language. We'll set the @@ -167,7 +167,7 @@ class SubscriptionService: # it can't be retrieved. Note that none of these are used unless # the address is completely new to us. password = generate(int(config.passwords.password_length)) - return add_member(mlist, subscriber, real_name, password, + return add_member(mlist, subscriber, display_name, password, delivery_mode, system_preferences.preferred_language, role) else: diff --git a/src/mailman/app/tests/test_bounces.py b/src/mailman/app/tests/test_bounces.py index be2c5cb78..d0d94df5e 100644 --- a/src/mailman/app/tests/test_bounces.py +++ b/src/mailman/app/tests/test_bounces.py @@ -21,6 +21,11 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestMaybeForward', + 'TestProbe', + 'TestSendProbe', + 'TestSendProbeNonEnglish', + 'TestVERP', ] @@ -212,7 +217,7 @@ Message-ID: <first> self.assertEqual(set(pendable.keys()), set(['member_id', 'message_id'])) # member_ids are pended as unicodes. - self.assertEqual(uuid.UUID(hex=pendable['member_id']), + self.assertEqual(uuid.UUID(hex=pendable['member_id']), self._member.member_id) self.assertEqual(pendable['message_id'], '<first>') @@ -264,10 +269,16 @@ Message-ID: <first> # Check the headers of the outer message. token = send_probe(self._member, self._msg) message = get_queue_messages('virgin')[0].msg - self.assertEqual(message['From'], + self.assertEqual(message['from'], 'test-bounces+{0}@example.com'.format(token)) - self.assertEqual(message['To'], 'anne@example.com') - self.assertEqual(message['Subject'], 'Test mailing list probe message') + self.assertEqual(message['to'], 'anne@example.com') + self.assertEqual(message['subject'], 'Test mailing list probe message') + + def test_no_precedence_header(self): + # Probe messages should not have a Precedence header (LP: #808821). + send_probe(self._member, self._msg) + message = get_queue_messages('virgin')[0].msg + self.assertEqual(message['precedence'], None) diff --git a/src/mailman/app/tests/test_notifications.py b/src/mailman/app/tests/test_notifications.py index 42f482582..8cce1be6f 100644 --- a/src/mailman/app/tests/test_notifications.py +++ b/src/mailman/app/tests/test_notifications.py @@ -50,7 +50,7 @@ class TestNotifications(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') self._mlist.welcome_message_uri = 'mailman:///welcome.txt' - self._mlist.real_name = 'Test List' + self._mlist.display_name = 'Test List' self.var_dir = tempfile.mkdtemp() config.push('template config', """\ [paths.testing] diff --git a/src/mailman/app/tests/test_subscriptions.py b/src/mailman/app/tests/test_subscriptions.py index 6ab386f7f..a63c9ac04 100644 --- a/src/mailman/app/tests/test_subscriptions.py +++ b/src/mailman/app/tests/test_subscriptions.py @@ -17,10 +17,11 @@ """Tests for the subscription service.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestJoin' ] @@ -63,6 +64,6 @@ class TestJoin(unittest.TestCase): try: self._service.join('test@example.com', 'bogus') except InvalidEmailAddressError as exc: - self.assertEqual(exc.address, 'bogus') + self.assertEqual(exc.email, 'bogus') else: raise AssertionError('InvalidEmailAddressError expected') diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 45ec8f194..7437f4790 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -21,7 +21,6 @@ header, and one that provides a *permalink* to the specific message object in the archive. This latter is appropriate for the message footer or for the RFC 5064 ``Archived-At:`` header. -Pipermail does not support a permalink, so that interface returns ``None``. Mailman defines a draft spec for how list servers and archivers can interoperate. @@ -38,9 +37,6 @@ interoperate. mhonarc http://lists.example.com/.../test@example.com http://lists.example.com/.../RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE - pipermail - http://www.example.com/pipermail/test@example.com - None prototype http://lists.example.com http://lists.example.com/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE @@ -49,25 +45,14 @@ interoperate. Sending the message to the archiver =================================== -The archiver is also able to archive the message. -:: - - >>> archivers['pipermail'].archive_message(mlist, msg) +The `prototype` archiver archives messages to a maildir. >>> import os - >>> from mailman.interfaces.archiver import IPipermailMailingList - >>> pckpath = os.path.join( - ... IPipermailMailingList(mlist).archive_dir(), - ... 'pipermail.pck') - >>> os.path.exists(pckpath) - True - -Note however that the prototype archiver can't archive messages. - >>> archivers['prototype'].archive_message(mlist, msg) - Traceback (most recent call last): - ... - NotImplementedError + >>> archive_path = os.path.join( + ... config.ARCHIVE_DIR, 'prototype', mlist.fqdn_listname, 'new') + >>> len(os.listdir(archive_path)) + 1 The Mail-Archive.com @@ -172,20 +157,17 @@ A MHonArc_ archiver is also available. Messages sent to a local MHonArc instance are added to its archive via a subprocess call. + >>> from mailman.testing.helpers import LogFileMark + >>> mark = LogFileMark('mailman.archiver') >>> archiver.archive_message(mlist, msg) - >>> archive_log = open(os.path.join(config.LOG_DIR, 'archiver')) - >>> try: - ... contents = archive_log.read() - ... finally: - ... archive_log.close() - >>> print 'LOG:', contents - LOG: ... /usr/bin/mhonarc -add - -dbfile /.../private/test@example.com.mbox/mhonarc.db - -outdir /.../mhonarc/test@example.com - -stderr /.../logs/mhonarc - -stdout /.../logs/mhonarc - -spammode -umask 022 - ... + >>> print 'LOG:', mark.readline() + LOG: ... /usr/bin/mhonarc + -add + -dbfile .../test@example.com.mbox/mhonarc.db + -outdir .../mhonarc/test@example.com + -stderr .../logs/mhonarc + -stdout .../logs/mhonarc -spammode -umask 022 + .. _`The Mail Archive`: http://www.mail-archive.com .. _MHonArc: http://www.mhonarc.org diff --git a/src/mailman/archiving/pipermail.py b/src/mailman/archiving/pipermail.py deleted file mode 100644 index 03dcd97f4..000000000 --- a/src/mailman/archiving/pipermail.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (C) 2007-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Pipermail archiver.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Pipermail', - ] - - -import os -import mailbox -import tempfile - -from zope.interface import implements -from zope.interface.interface import adapter_hooks - -from mailman.config import config -from mailman.interfaces.archiver import IArchiver, IPipermailMailingList -from mailman.interfaces.mailinglist import IMailingList -from mailman.utilities.filesystem import makedirs -from mailman.utilities.string import expand - -from mailman.Archiver.HyperArch import HyperArchive - - - -class PipermailMailingListAdapter: - """An adapter for MailingList objects to work with Pipermail.""" - - implements(IPipermailMailingList) - - def __init__(self, mlist): - self._mlist = mlist - - def __getattr__(self, name): - return getattr(self._mlist, name) - - def archive_dir(self): - """See `IPipermailMailingList`.""" - if self._mlist.archive_private: - basedir = config.PRIVATE_ARCHIVE_FILE_DIR - else: - basedir = config.PUBLIC_ARCHIVE_FILE_DIR - # Make sure the archive directory exists. - archive_dir = os.path.join(basedir, self._mlist.fqdn_listname) - makedirs(archive_dir) - return archive_dir - - -def adapt_mailing_list_for_pipermail(iface, obj): - """Adapt `IMailingLists` to `IPipermailMailingList`. - - :param iface: The interface to adapt to. - :type iface: `zope.interface.Interface` - :param obj: The object being adapted. - :type obj: any object - :return: An `IPipermailMailingList` instance if adaptation succeeded or - None if it didn't. - """ - return (PipermailMailingListAdapter(obj) - if IMailingList.providedBy(obj) and iface is IPipermailMailingList - else None) - -adapter_hooks.append(adapt_mailing_list_for_pipermail) - - - -class Pipermail: - """The stock Pipermail archiver.""" - - implements(IArchiver) - - name = 'pipermail' - - @staticmethod - def list_url(mlist): - """See `IArchiver`.""" - if mlist.archive_private: - return mlist.script_url('private') + '/index.html' - else: - return expand(config.archiver.pipermail.base_url, - dict(listname=mlist.fqdn_listname, - hostname=mlist.domain.url_host, - fqdn_listname=mlist.fqdn_listname, - )) - - @staticmethod - def permalink(mlist, message): - """See `IArchiver`.""" - # Not currently implemented. - return None - - @staticmethod - def archive_message(mlist, message): - """See `IArchiver`.""" - fd, path = tempfile.mkstemp('.mbox') - os.close(fd) - try: - mbox = mailbox.mbox(path, create=True) - mbox.add(message) - finally: - mbox.close() - h = HyperArchive(IPipermailMailingList(mlist)) - try: - h.processUnixMailbox(path) - finally: - h.close() - os.remove(path) - # There's no good way to know the url for the archived message. - return None diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 55d78074e..453c6c770 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -25,11 +25,22 @@ __all__ = [ ] +import os +import errno +import logging + +from datetime import timedelta +from mailbox import Maildir from urlparse import urljoin + +from flufl.lock import Lock, TimeOutError from zope.interface import implements +from mailman.config import config from mailman.interfaces.archiver import IArchiver +log = logging.getLogger('mailman.error') + class Prototype: @@ -61,5 +72,49 @@ class Prototype: @staticmethod def archive_message(mlist, message): - """See `IArchiver`.""" - raise NotImplementedError + """See `IArchiver`. + + This archiver saves messages into a maildir. + """ + archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype') + try: + os.makedirs(archive_dir, 0775) + except OSError as error: + # If this already exists, then we're fine + if error.errno != errno.EEXIST: + raise + + # Maildir will throw an error if the directories are partially created + # (for instance the toplevel exists but cur, new, or tmp do not) + # therefore we don't create the toplevel as we did above. + list_dir = os.path.join(archive_dir, mlist.fqdn_listname) + mailbox = Maildir(list_dir, create=True, factory=None) + lock_file = os.path.join( + config.LOCK_DIR, '{0}-maildir.lock'.format(mlist.fqdn_listname)) + + # Lock the maildir as Maildir.add() is not threadsafe. Don't use the + # context manager because it's not an error if we can't acquire the + # archiver lock. We'll just log the problem and continue. + # + # XXX 2012-03-14 BAW: When we extend the chain/pipeline architecture + # to other runners, e.g. the archive runner, it would be better to let + # any TimeOutError propagate up. That would cause the message to be + # re-queued and tried again later, rather than being discarded as + # happens now below. + lock = Lock(lock_file) + try: + lock.lock(timeout=timedelta(seconds=1)) + # Add the message to the maildir. The return value could be used + # to construct the file path if necessary. E.g. + # + # os.path.join(archive_dir, mlist.fqdn_listname, 'new', + # message_key) + mailbox.add(message) + except TimeOutError: + # Log the error and go on. + log.error('Unable to acquire prototype archiver lock for {0}, ' + 'discarding: {1}'.format( + mlist.fqdn_listname, + message.get('message-id', 'n/a'))) + finally: + lock.unlock(unconditionally=True) diff --git a/src/mailman/pipeline/__init__.py b/src/mailman/archiving/tests/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/mailman/pipeline/__init__.py +++ b/src/mailman/archiving/tests/__init__.py diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py new file mode 100644 index 000000000..29f6ba1cb --- /dev/null +++ b/src/mailman/archiving/tests/test_prototype.py @@ -0,0 +1,174 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the prototype archiver.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPrototypeArchiver', + ] + + +import os +import shutil +import tempfile +import unittest +import threading + +from email import message_from_file +from flufl.lock import Lock + +from mailman.app.lifecycle import create_list +from mailman.archiving.prototype import Prototype +from mailman.config import config +from mailman.testing.helpers import LogFileMark +from mailman.testing.helpers import ( + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer +from mailman.utilities.email import add_message_hash + + +class TestPrototypeArchiver(unittest.TestCase): + """Test the prototype archiver.""" + + layer = ConfigLayer + + def setUp(self): + # Create a fake mailing list and message object + self._msg = mfs("""\ +To: test@example.com +From: anne@example.com +Subject: Testing the test list +Message-ID: <ant> +X-Message-ID-Hash: MS6QLWERIJLGCRF44J7USBFDELMNT2BW + +Tests are better than no tests +but the water deserves to be swum. +""") + self._mlist = create_list('test@example.com') + config.db.commit() + # Set up a temporary directory for the prototype archiver so that it's + # easier to clean up. + self._tempdir = tempfile.mkdtemp() + config.push('prototype', """ + [paths.testing] + archive_dir: {0} + """.format(self._tempdir)) + # Capture the structure of a maildir. + self._expected_dir_structure = set( + (os.path.join(config.ARCHIVE_DIR, path) for path in ( + 'prototype', + os.path.join('prototype', self._mlist.fqdn_listname), + os.path.join('prototype', self._mlist.fqdn_listname, 'cur'), + os.path.join('prototype', self._mlist.fqdn_listname, 'new'), + os.path.join('prototype', self._mlist.fqdn_listname, 'tmp'), + ))) + self._expected_dir_structure.add(config.ARCHIVE_DIR) + + def tearDown(self): + shutil.rmtree(self._tempdir) + config.pop('prototype') + + def _find(self, path): + all_filenames = set() + for dirpath, dirnames, filenames in os.walk(path): + if not isinstance(dirpath, unicode): + dirpath = unicode(dirpath) + all_filenames.add(dirpath) + for filename in filenames: + new_filename = filename + if not isinstance(filename, unicode): + new_filename = unicode(filename) + all_filenames.add(os.path.join(dirpath, new_filename)) + return all_filenames + + def test_archive_maildir_created(self): + # Archiving a message to the prototype archiver should create the + # expected directory structure. + Prototype.archive_message(self._mlist, self._msg) + all_filenames = self._find(config.ARCHIVE_DIR) + # Check that the directory structure has been created and we have one + # more file (the archived message) than expected directories. + archived_messages = all_filenames - self._expected_dir_structure + self.assertEqual(len(archived_messages), 1) + self.assertTrue( + archived_messages.pop().startswith( + os.path.join(config.ARCHIVE_DIR, 'prototype', + self._mlist.fqdn_listname, 'new'))) + + def test_archive_maildir_existence_does_not_raise(self): + # Archiving a second message does not cause an EEXIST to be raised + # when a second message is archived. + new_dir = None + Prototype.archive_message(self._mlist, self._msg) + for directory in ('cur', 'new', 'tmp'): + path = os.path.join(config.ARCHIVE_DIR, 'prototype', + self._mlist.fqdn_listname, directory) + if directory == 'new': + new_dir = path + self.assertTrue(os.path.isdir(path)) + # There should be one message in the 'new' directory. + self.assertEqual(len(os.listdir(new_dir)), 1) + # Archive a second message. If an exception occurs, let it fail the + # test. Afterward, two messages should be in the 'new' directory. + del self._msg['message-id'] + del self._msg['x-message-id-hash'] + self._msg['Message-ID'] = '<bee>' + add_message_hash(self._msg) + Prototype.archive_message(self._mlist, self._msg) + self.assertEqual(len(os.listdir(new_dir)), 2) + + def test_archive_lock_used(self): + # Test that locking the maildir when adding works as a failure here + # could mean we lose mail. + lock_file = os.path.join( + config.LOCK_DIR, '{0}-maildir.lock'.format( + self._mlist.fqdn_listname)) + with Lock(lock_file): + # Acquire the archiver lock, then make sure the archiver logs the + # fact that it could not acquire the lock. + archive_thread = threading.Thread( + target=Prototype.archive_message, + args=(self._mlist, self._msg)) + mark = LogFileMark('mailman.error') + archive_thread.run() + # Test that the archiver output the correct error. + line = mark.readline() + # XXX 2012-03-15 BAW: we really should remove timestamp prefixes + # from the loggers when under test. + self.assertTrue(line.endswith( + 'Unable to acquire prototype archiver lock for {0}, ' + 'discarding: {1}\n'.format( + self._mlist.fqdn_listname, + self._msg.get('message-id')))) + # Check that the message didn't get archived. + created_files = self._find(config.ARCHIVE_DIR) + self.assertEqual(self._expected_dir_structure, created_files) + + def test_prototype_archiver_good_path(self): + # Verify the good path; the message gets archived. + Prototype.archive_message(self._mlist, self._msg) + new_path = os.path.join( + config.ARCHIVE_DIR, 'prototype', self._mlist.fqdn_listname, 'new') + archived_messages = list(os.listdir(new_path)) + self.assertEqual(len(archived_messages), 1) + # Check that the email has been added. + with open(os.path.join(new_path, archived_messages[0])) as fp: + archived_message = message_from_file(fp) + self.assertEqual(self._msg.as_string(), archived_message.as_string()) diff --git a/src/mailman/bin/arch.py b/src/mailman/bin/arch.py deleted file mode 100644 index 64fd235aa..000000000 --- a/src/mailman/bin/arch.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -from __future__ import absolute_import, unicode_literals - -import os -import sys -import errno -import shutil -import optparse - -from flufl.lock import Lock - -from mailman.Archiver.HyperArch import HyperArchive -from mailman.Defaults import hours -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%%prog [options] listname [mbox] - -Rebuild a list's archive. - -Use this command to rebuild the archives for a mailing list. You may want to -do this if you edit some messages in an archive, or remove some messages from -an archive. - -Where 'mbox' is the path to a list's complete mbox archive. Usually this will -be some path in the archives/private directory. For example: - -% bin/arch mylist archives/private/mylist.mbox/mylist.mbox - -'mbox' is optional. If it is missing, it is calculated from the listname. -""")) - parser.add_option('-q', '--quiet', - dest='verbose', default=True, action='store_false', - help=_('Make the archiver output less verbose')) - parser.add_option('--wipe', - default=False, action='store_true', - help=_("""\ -First wipe out the original archive before regenerating. You usually want to -specify this argument unless you're generating the archive in chunks.""")) - parser.add_option('-s', '--start', - default=None, type='int', metavar='N', - help=_("""\ -Start indexing at article N, where article 0 is the first in the mbox. -Defaults to 0.""")) - parser.add_option('-e', '--end', - default=None, type='int', metavar='M', - help=_("""\ -End indexing at article M. This script is not very efficient with respect to -memory management, and for large archives, it may not be possible to index the -mbox entirely. For that reason, you can specify the start and end article -numbers.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if len(args) < 1: - parser.print_help() - print >> sys.stderr, _('listname is required') - sys.exit(1) - if len(args) > 2: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - listname = args[0].lower().strip() - if len(args) < 2: - mbox = None - else: - mbox = args[1] - - # Open the mailing list object - mlist = config.list_manager.get(listname) - if mlist is None: - parser.error(_('No such list: $listname')) - if mbox is None: - mbox = mlist.ArchiveFileName() - - # Set the default language to the list's preferred language. - _.default = mlist.preferred_language - # Lay claim to the archive's lock file. This is so no other post can - # mess up the archive while we're processing it. Try to pick a - # suitably long period of time for the lock lifetime even though we - # really don't know how long it will take. - # - # XXX processUnixMailbox() should refresh the lock. - lock_path = os.path.join(mlist.data_path, '.archiver.lck') - with Lock(lock_path, lifetime=int(hours(3))): - # Try to open mbox before wiping old archive. - try: - fp = open(mbox) - except IOError, e: - if e.errno == errno.ENOENT: - print >> sys.stderr, _('Cannot open mbox file: $mbox') - else: - print >> sys.stderr, e - sys.exit(1) - # Maybe wipe the old archives - if opts.wipe: - if mlist.scrub_nondigest: - # TK: save the attachments dir because they are not in mbox - saved = False - atchdir = os.path.join(mlist.archive_dir(), 'attachments') - savedir = os.path.join(mlist.archive_dir() + '.mbox', - 'attachments') - try: - os.rename(atchdir, savedir) - saved = True - except OSError, e: - if e.errno != errno.ENOENT: - raise - shutil.rmtree(mlist.archive_dir()) - if mlist.scrub_nondigest and saved: - os.renames(savedir, atchdir) - - archiver = HyperArchive(mlist) - archiver.VERBOSE = opts.verbose - try: - archiver.processUnixMailbox(fp, opts.start, opts.end) - finally: - archiver.close() - fp.close() diff --git a/src/mailman/bin/check_perms.py b/src/mailman/bin/check_perms.py deleted file mode 100644 index 6e0d761d9..000000000 --- a/src/mailman/bin/check_perms.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import pwd -import grp -import errno -import optparse - -from stat import * - -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -# XXX Need to check the archives/private/*/database/* files - - - -class State: - FIX = False - VERBOSE = False - ERRORS = 0 - -STATE = State() - -DIRPERMS = S_ISGID | S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH -QFILEPERMS = S_ISGID | S_IRWXU | S_IRWXG -PYFILEPERMS = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH -ARTICLEFILEPERMS = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP -MBOXPERMS = S_IRGRP | S_IWGRP | S_IRUSR | S_IWUSR -PRIVATEPERMS = QFILEPERMS - - - -def statmode(path): - return os.stat(path).st_mode - - -def statgidmode(path): - stat = os.stat(path) - return stat.st_mode, stat.st_gid - - -seen = {} - -# libc's getgrgid re-opens /etc/group each time :( -_gidcache = {} - -def getgrgid(gid): - data = _gidcache.get(gid) - if data is None: - data = grp.getgrgid(gid) - _gidcache[gid] = data - return data - - - -def checkwalk(arg, dirname, names): - # Short-circuit duplicates - if seen.has_key(dirname): - return - seen[dirname] = True - for name in names: - path = os.path.join(dirname, name) - if arg.VERBOSE: - print _(' checking gid and mode for $path') - try: - mode, gid = statgidmode(path) - except OSError, e: - if e.errno != errno.ENOENT: raise - continue - if gid != MAILMAN_GID: - try: - groupname = getgrgid(gid)[0] - except KeyError: - groupname = '<anon gid %d>' % gid - arg.ERRORS += 1 - print _( - '$path bad group (has: $groupname, expected $MAILMAN_GROUP)'), - if STATE.FIX: - print _('(fixing)') - os.chown(path, -1, MAILMAN_GID) - else: - print - # Most directories must be at least rwxrwsr-x. - # The private archive directory and database directory must be at - # least rwxrws---. Their 'other' permissions are checked in - # checkarchives() and checkarchivedbs() below. Their 'user' and - # 'group' permissions are checked here. - # The directories under qfiles should be rwxrws---. Their 'user' and - # 'group' permissions are checked here. Their 'other' permissions - # aren't checked. - private = config.PRIVATE_ARCHIVE_FILE_DIR - if path == private or ( - os.path.commonprefix((path, private)) == private - and os.path.split(path)[1] == 'database'): - # then... - targetperms = PRIVATEPERMS - elif (os.path.commonprefix((path, config.QUEUE_DIR)) - == config.QUEUE_DIR): - targetperms = QFILEPERMS - else: - targetperms = DIRPERMS - octperms = oct(targetperms) - if S_ISDIR(mode) and (mode & targetperms) != targetperms: - arg.ERRORS += 1 - print _('directory permissions must be $octperms: $path'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | targetperms) - else: - print - elif os.path.splitext(path)[1] in ('.py', '.pyc', '.pyo'): - octperms = oct(PYFILEPERMS) - if mode & PYFILEPERMS != PYFILEPERMS: - print _('source perms must be $octperms: $path'), - arg.ERRORS += 1 - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | PYFILEPERMS) - else: - print - elif path.endswith('-article'): - # Article files must be group writeable - octperms = oct(ARTICLEFILEPERMS) - if mode & ARTICLEFILEPERMS != ARTICLEFILEPERMS: - print _('article db files must be $octperms: $path'), - arg.ERRORS += 1 - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | ARTICLEFILEPERMS) - else: - print - - - -def checkall(): - # first check PREFIX - if STATE.VERBOSE: - prefix = config.PREFIX - print _('checking mode for $prefix') - dirs = {} - for d in (config.PREFIX, config.EXEC_PREFIX, config.VAR_PREFIX, - config.LOG_DIR): - dirs[d] = True - for d in dirs.keys(): - try: - mode = statmode(d) - except OSError, e: - if e.errno != errno.ENOENT: raise - print _('WARNING: directory does not exist: $d') - continue - if (mode & DIRPERMS) != DIRPERMS: - STATE.ERRORS += 1 - print _('directory must be at least 02775: $d'), - if STATE.FIX: - print _('(fixing)') - os.chmod(d, mode | DIRPERMS) - else: - print - # check all subdirs - os.path.walk(d, checkwalk, STATE) - - - -def checkarchives(): - private = config.PRIVATE_ARCHIVE_FILE_DIR - if STATE.VERBOSE: - print _('checking perms on $private') - # private archives must not be other readable - mode = statmode(private) - if mode & S_IROTH: - STATE.ERRORS += 1 - print _('$private must not be other-readable'), - if STATE.FIX: - print _('(fixing)') - os.chmod(private, mode & ~S_IROTH) - else: - print - # In addition, on a multiuser system you may want to hide the private - # archives so other users can't read them. - if mode & S_IXOTH: - print _("""\ -Warning: Private archive directory is other-executable (o+x). - This could allow other users on your system to read private archives. - If you're on a shared multiuser system, you should consult the - installation manual on how to fix this.""") - - - -def checkmboxfile(mboxdir): - absdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, mboxdir) - for f in os.listdir(absdir): - if not f.endswith('.mbox'): - continue - mboxfile = os.path.join(absdir, f) - mode = statmode(mboxfile) - if (mode & MBOXPERMS) != MBOXPERMS: - STATE.ERRORS = STATE.ERRORS + 1 - print _('mbox file must be at least 0660:'), mboxfile - if STATE.FIX: - print _('(fixing)') - os.chmod(mboxfile, mode | MBOXPERMS) - else: - print - - - -def checkarchivedbs(): - # The archives/private/listname/database file must not be other readable - # or executable otherwise those files will be accessible when the archives - # are public. That may not be a horrible breach, but let's close this off - # anyway. - for dir in os.listdir(config.PRIVATE_ARCHIVE_FILE_DIR): - if dir.endswith('.mbox'): - checkmboxfile(dir) - dbdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, dir, 'database') - try: - mode = statmode(dbdir) - except OSError, e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise - continue - if mode & S_IRWXO: - STATE.ERRORS += 1 - print _('$dbdir "other" perms must be 000'), - if STATE.FIX: - print _('(fixing)') - os.chmod(dbdir, mode & ~S_IRWXO) - else: - print - - - -def checkcgi(): - cgidir = os.path.join(config.EXEC_PREFIX, 'cgi-bin') - if STATE.VERBOSE: - print _('checking cgi-bin permissions') - exes = os.listdir(cgidir) - for f in exes: - path = os.path.join(cgidir, f) - if STATE.VERBOSE: - print _(' checking set-gid for $path') - mode = statmode(path) - if mode & S_IXGRP and not mode & S_ISGID: - STATE.ERRORS += 1 - print _('$path must be set-gid'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | S_ISGID) - else: - print - - - -def checkmail(): - wrapper = os.path.join(config.WRAPPER_DIR, 'mailman') - if STATE.VERBOSE: - print _('checking set-gid for $wrapper') - mode = statmode(wrapper) - if not mode & S_ISGID: - STATE.ERRORS += 1 - print _('$wrapper must be set-gid'), - if STATE.FIX: - print _('(fixing)') - os.chmod(wrapper, mode | S_ISGID) - - - -def checkadminpw(): - for pwfile in (os.path.join(config.DATA_DIR, 'adm.pw'), - os.path.join(config.DATA_DIR, 'creator.pw')): - targetmode = S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP - if STATE.VERBOSE: - print _('checking permissions on $pwfile') - try: - mode = statmode(pwfile) - except OSError, e: - if e.errno != errno.ENOENT: - raise - return - if mode != targetmode: - STATE.ERRORS += 1 - octmode = oct(mode) - print _('$pwfile permissions must be exactly 0640 (got $octmode)'), - if STATE.FIX: - print _('(fixing)') - os.chmod(pwfile, targetmode) - else: - print - - -def checkmta(): - if config.MTA: - modname = 'mailman.MTA.' + config.MTA - __import__(modname) - try: - sys.modules[modname].checkperms(STATE) - except AttributeError: - pass - - - -def checkdata(): - targetmode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP - checkfiles = ('config.pck', 'config.pck.last', - 'config.db', 'config.db.last', - 'next-digest', 'next-digest-topics', - 'digest.mbox', 'pending.pck', - 'request.db', 'request.db.tmp') - if STATE.VERBOSE: - print _('checking permissions on list data') - for dir in os.listdir(config.LIST_DATA_DIR): - for file in checkfiles: - path = os.path.join(config.LIST_DATA_DIR, dir, file) - if STATE.VERBOSE: - print _(' checking permissions on: $path') - try: - mode = statmode(path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - continue - if (mode & targetmode) != targetmode: - STATE.ERRORS += 1 - print _('file permissions must be at least 660: $path'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | targetmode) - else: - print - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Check the permissions of all Mailman files. With no options, just report the -permission and ownership problems found.""")) - parser.add_option('-f', '--fix', - default=False, action='store_true', help=_("""\ -Fix all permission and ownership problems found. With this option, you must -run check_perms as root.""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', - help=_('Produce more verbose output')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def main(): - global MAILMAN_USER, MAILMAN_GROUP, MAILMAN_UID, MAILMAN_GID - - parser, opts, args = parseargs() - STATE.FIX = opts.fix - STATE.VERBOSE = opts.verbose - - config.load(opts.config) - - MAILMAN_USER = config.MAILMAN_USER - MAILMAN_GROUP = config.MAILMAN_GROUP - # Let KeyErrors percolate - MAILMAN_GID = grp.getgrnam(MAILMAN_GROUP).gr_gid - MAILMAN_UID = pwd.getpwnam(MAILMAN_USER).pw_uid - - checkall() - checkarchives() - checkarchivedbs() - checkcgi() - checkmail() - checkdata() - checkadminpw() - checkmta() - - if not STATE.ERRORS: - print _('No problems found') - else: - print _('Problems found:'), STATE.ERRORS - print _('Re-run as $MAILMAN_USER (or root) with -f flag to fix') - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/cleanarch.py b/src/mailman/bin/cleanarch.py deleted file mode 100644 index 48b96d191..000000000 --- a/src/mailman/bin/cleanarch.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Clean up an .mbox archive file.""" - -import re -import sys -import mailbox -import optparse - -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -cre = re.compile(mailbox.UnixMailbox._fromlinepattern) -# From RFC 2822, a header field name must contain only characters from 33-126 -# inclusive, excluding colon. I.e. from oct 41 to oct 176 less oct 072. Must -# use re.match() so that it's anchored at the beginning of the line. -fre = re.compile(r'[\041-\071\073-\176]+') - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] < inputfile > outputfile - -The archiver looks for Unix-From lines separating messages in an mbox archive -file. For compatibility, it specifically looks for lines that start with -'From ' -- i.e. the letters capital-F, lowercase-r, o, m, space, ignoring -everything else on the line. - -Normally, any lines that start 'From ' in the body of a message should be -escaped such that a > character is actually the first on a line. It is -possible though that body lines are not actually escaped. This script -attempts to fix these by doing a stricter test of the Unix-From lines. Any -lines that start From ' but do not pass this stricter test are escaped with a -'>' character.""")) - parser.add_option('-q', '--quiet', - default=False, action='store_true', help=_("""\ -Don't print changed line information to standard error.""")) - parser.add_option('-s', '--status', - default=-1, type='int', help=_("""\ -Print a '#' character for every n lines processed. With a number less than or -equal to zero, suppress the '#' characters.""")) - parser.add_option('-n', '--dry-run', - default=False, action='store_true', help=_("""\ -Don't actually output anything.""")) - opts, args = parser.parser_args() - if args: - parser.print_error(_('Unexpected arguments')) - return parser, opts, args - - - -def escape_line(line, lineno, quiet, output): - if output: - sys.stdout.write('>' + line) - if not quiet: - print >> sys.stderr, _('Unix-From line changed: $lineno') - print >> sys.stderr, line[:-1] - - - -def main(): - parser, opts, args = parseargs() - - lineno = 0 - statuscnt = 0 - messages = 0 - prevline = None - while True: - lineno += 1 - line = sys.stdin.readline() - if not line: - break - if line.startswith('From '): - if cre.match(line): - # This is a real Unix-From line. But it could be a message - # /about/ Unix-From lines, so as a second order test, make - # sure there's at least one RFC 2822 header following - nextline = sys.stdin.readline() - lineno += 1 - if not nextline: - # It was the last line of the mbox, so it couldn't have - # been a Unix-From - escape_line(line, lineno, quiet, output) - break - fieldname = nextline.split(':', 1) - if len(fieldname) < 2 or not fre.match(nextline): - # The following line was not a header, so this wasn't a - # valid Unix-From - escape_line(line, lineno, quiet, output) - if output: - sys.stdout.write(nextline) - else: - # It's a valid Unix-From line - messages += 1 - if output: - # Before we spit out the From_ line, make sure the - # previous line was blank. - if prevline is not None and prevline != '\n': - sys.stdout.write('\n') - sys.stdout.write(line) - sys.stdout.write(nextline) - else: - # This is a bogus Unix-From line - escape_line(line, lineno, quiet, output) - elif output: - # Any old line - sys.stdout.write(line) - if status > 0 and (lineno % status) == 0: - sys.stderr.write('#') - statuscnt += 1 - if statuscnt > 50: - print >> sys.stderr - statuscnt = 0 - prevline = line - print >> sys.stderr, _('%(messages)d messages found') diff --git a/src/mailman/bin/disabled.py b/src/mailman/bin/disabled.py index 4d7167417..c7cab4829 100644 --- a/src/mailman/bin/disabled.py +++ b/src/mailman/bin/disabled.py @@ -54,7 +54,7 @@ def who_callback(option, opt, value, parser): dest.discard(MemberAdaptor.BYBOUNCE) elif opt in ('-a', '--all'): dest.update(ALL) - +x5o def parseargs(): parser = optparse.OptionParser(version=MAILMAN_VERSION, diff --git a/src/mailman/bin/nightly_gzip.py b/src/mailman/bin/nightly_gzip.py deleted file mode 100644 index 2d1c5b8a1..000000000 --- a/src/mailman/bin/nightly_gzip.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import optparse - -try: - import gzip -except ImportError: - sys.exit(0) - -from mailman import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] [listname ...] - -Re-generate the Pipermail gzip'd archive flat files.""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', - help=_("Print each file as it's being gzip'd")) - parser.add_option('-z', '--level', - default=6, type='int', - help=_('Specifies the compression level')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if opts.level < 1 or opts.level > 9: - parser.print_help() - print >> sys.stderr, _('Illegal compression level: $opts.level') - sys.exit(1) - return opts, args, parser - - - -def compress(txtfile, opts): - if opts.verbose: - print _("gzip'ing: $txtfile") - infp = outfp = None - try: - infp = open(txtfile) - outfp = gzip.open(txtfile + '.gz', 'wb', opts.level) - outfp.write(infp.read()) - finally: - if outfp: - outfp.close() - if infp: - infp.close() - - - -def main(): - opts, args, parser = parseargs() - initialize(opts.config) - - if config.ARCHIVE_TO_MBOX not in (1, 2) or config.GZIP_ARCHIVE_TXT_FILES: - # We're only going to run the nightly archiver if messages are - # archived to the mbox, and the gzip file is not created on demand - # (i.e. for every individual post). This is the normal mode of - # operation. - return - - # Process all the specified lists - for listname in set(args or config.list_manager.names): - mlist = MailList.MailList(listname, lock=False) - if not mlist.archive: - continue - dir = mlist.archive_dir() - try: - allfiles = os.listdir(dir) - except OSError: - # Has the list received any messages? If not, last_post_time will - # be zero, so it's not really a bogus archive dir. - if mlist.last_post_time > 0: - print _('List $listname has a bogus archive_directory: $dir') - continue - if opts.verbose: - print _('Processing list: $listname') - files = [] - for f in allfiles: - if os.path.splitext(f)[1] <> '.txt': - continue - # stat both the .txt and .txt.gz files and append them only if - # the former is newer than the latter. - txtfile = os.path.join(dir, f) - gzpfile = txtfile + '.gz' - txt_mtime = os.path.getmtime(txtfile) - try: - gzp_mtime = os.path.getmtime(gzpfile) - except OSError: - gzp_mtime = -1 - if txt_mtime > gzp_mtime: - files.append(txtfile) - for f in files: - compress(f, opts) diff --git a/src/mailman/bin/show_qfiles.py b/src/mailman/bin/show_qfiles.py deleted file mode 100644 index 08f5fb04e..000000000 --- a/src/mailman/bin/show_qfiles.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2006-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys - -from cPickle import load - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.options import Options - - - -class ScriptOptions(Options): - usage = _(""" -%%prog [options] qfiles ... - -Show the contents of one or more Mailman queue files.""") - - def add_options(self): - super(ScriptOptions, self).add_options() - self.parser.add_option( - '-q', '--quiet', - default=False, action='store_true', - help=_("Don't print 'helpful' message delimiters.")) - self.parser.add_option( - '-s', '--summary', - default=False, action='store_true', - help=_('Show a summary of queue files.')) - - - -def main(): - options = ScriptOptions() - options.initialize() - - if options.options.summary: - queue_totals = {} - files_by_queue = {} - for switchboard in config.switchboards.values(): - total = 0 - file_mappings = {} - for filename in os.listdir(switchboard.queue_directory): - base, ext = os.path.splitext(filename) - file_mappings[ext] = file_mappings.get(ext, 0) + 1 - total += 1 - files_by_queue[switchboard.queue_directory] = file_mappings - queue_totals[switchboard.queue_directory] = total - # Sort by queue name. - for queue_directory in sorted(files_by_queue): - total = queue_totals[queue_directory] - print queue_directory - print _('\tfile count: $total') - file_mappings = files_by_queue[queue_directory] - for ext in sorted(file_mappings): - print '\t{0}: {1}'.format(ext, file_mappings[ext]) - return - # No summary. - for filename in options.arguments: - if not options.options.quiet: - print '====================>', filename - with open(filename) as fp: - if filename.endswith('.pck'): - msg = load(fp) - data = load(fp) - if data.get('_parsemsg'): - sys.stdout.write(msg) - else: - sys.stdout.write(msg.as_string()) - else: - sys.stdout.write(fp.read()) - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/update.py b/src/mailman/bin/update.py deleted file mode 100644 index 3beda2d97..000000000 --- a/src/mailman/bin/update.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import md5 -import sys -import time -import email -import errno -import shutil -import cPickle -import marshal -import optparse - -from flufl.lock import TimeOutError - -from mailman import MailList -from mailman import Message -from mailman import Pending -from mailman import Utils -from mailman import version -from mailman.MemberAdaptor import BYBOUNCE, ENABLED -from mailman.OldStyleMemberships import OldStyleMemberships -from mailman.Queue.Switchboard import Switchboard -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.utilities.filesystem import makedirs - - -FRESH = 0 -NOTFRESH = -1 - - - -def parseargs(): - parser = optparse.OptionParser(version=version.MAILMAN_VERSION, - usage=_("""\ -Perform all necessary upgrades. - -%prog [options]""")) - parser.add_option('-f', '--force', - default=False, action='store_true', help=_("""\ -Force running the upgrade procedures. Normally, if the version number of the -installed Mailman matches the current version number (or a 'downgrade' is -detected), nothing will be done.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def calcversions(): - # Returns a tuple of (lastversion, thisversion). If the last version - # could not be determined, lastversion will be FRESH or NOTFRESH, - # depending on whether this installation appears to be fresh or not. The - # determining factor is whether there are files in the $var_prefix/logs - # subdir or not. The version numbers are HEX_VERSIONs. - # - # See if we stored the last updated version - lastversion = None - thisversion = version.HEX_VERSION - try: - fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version')) - data = fp.read() - fp.close() - lastversion = int(data, 16) - except (IOError, ValueError): - pass - # - # try to figure out if this is a fresh install - if lastversion is None: - lastversion = FRESH - try: - if os.listdir(config.LOG_DIR): - lastversion = NOTFRESH - except OSError: - pass - return (lastversion, thisversion) - - - -def makeabs(relpath): - return os.path.join(config.PREFIX, relpath) - - -def make_varabs(relpath): - return os.path.join(config.VAR_PREFIX, relpath) - - - -def move_language_templates(mlist): - listname = mlist.internal_name() - print _('Fixing language templates: $listname') - # Mailman 2.1 has a new cascading search for its templates, defined and - # described in Utils.py:maketext(). Putting templates in the top level - # templates/ subdir or the lists/<listname> subdir is deprecated and no - # longer searched.. - # - # What this means is that most templates can live in the global templates/ - # subdirectory, and only needs to be copied into the list-, vhost-, or - # site-specific language directories when needed. - # - # Also, by default all standard (i.e. English) templates must now live in - # the templates/en directory. This update cleans up all the templates, - # deleting more-specific duplicates (as calculated by md5 checksums) in - # favor of more-global locations. - # - # First, get rid of any lists/<list> template or lists/<list>/en template - # that is identical to the global templates/* default. - for gtemplate in os.listdir(os.path.join(config.TEMPLATE_DIR, 'en')): - # BAW: get rid of old templates, e.g. admlogin.txt and - # handle_opts.html - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - # No global template - continue - gcksum = md5.new(fp.read()).digest() - fp.close() - # Match against the lists/<list>/* template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate)) - # Match against the lists/<list>/*.prev template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - # Match against the lists/<list>/en/* templates - try: - fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) - # Match against the templates/* template - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(config.TEMPLATE_DIR, gtemplate)) - # Match against the templates/*.prev template - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(config.TEMPLATE_DIR, - gtemplate + '.prev')) - - - -def situate_list(listname): - # This turns the directory called 'listname' into a directory called - # 'listname@domain'. Start by finding out what the domain should be. - # A list's domain is its email host. - mlist = MailList.MailList(listname, lock=False, check_version=False) - fullname = mlist.fqdn_listname - oldpath = os.path.join(config.VAR_PREFIX, 'lists', listname) - newpath = os.path.join(config.VAR_PREFIX, 'lists', fullname) - if os.path.exists(newpath): - print >> sys.stderr, _('WARNING: could not situate list: $listname') - else: - os.rename(oldpath, newpath) - print _('situated list $listname to $fullname') - return fullname - - - -def dolist(listname): - mlist = MailList.MailList(listname, lock=False) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: $listname') - return 1 - # Sanity check the invariant that every BYBOUNCE disabled member must have - # bounce information. Some earlier betas broke this. BAW: we're - # submerging below the MemberAdaptor interface, so skip this if we're not - # using OldStyleMemberships. - if isinstance(mlist._memberadaptor, OldStyleMemberships): - noinfo = {} - for addr, (reason, when) in mlist.delivery_status.items(): - if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): - noinfo[addr] = reason, when - # What to do about these folks with a BYBOUNCE delivery status and no - # bounce info? This number should be very small, and I think it's - # fine to simple re-enable them and let the bounce machinery - # re-disable them if necessary. - n = len(noinfo) - if n > 0: - print _( - 'Resetting $n BYBOUNCEs disabled addrs with no bounce info') - for addr in noinfo.keys(): - mlist.setDeliveryStatus(addr, ENABLED) - - mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) - mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, - listname)) - o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) - o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) - html_dir = o_pri_mbox_file - o_html_dir = makeabs('public_html/archives/%s' % (listname)) - # Make the mbox directory if it's not there. - if not os.path.exists(mbox_dir): - makedirs(mbox_dir) - else: - # This shouldn't happen, but hey, just in case - if not os.path.isdir(mbox_dir): - print _("""\ -For some reason, $mbox_dir exists as a file. This won't work with b6, so I'm -renaming it to ${mbox_dir}.tmp and proceeding.""") - os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) - makedirs(mbox_dir) - # Move any existing mboxes around, but watch out for both a public and a - # private one existing - if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): - if mlist.archive_private: - print _("""\ - -$listname has both public and private mbox archives. Since this list -currently uses private archiving, I'm installing the private mbox archive -- -$o_pri_mbox_file -- as the active archive, and renaming - $o_pub_mbox_file -to - ${o_pub_mbox_file}.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, - o_pub_mbox_file) - os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) - else: - print _("""\ -$mlist._internal_name has both public and private mbox archives. Since this -list currently uses public archiving, I'm installing the public mbox file -archive file ($o_pub_mbox_file) as the active one, and renaming -$o_pri_mbox_file to ${o_pri_mbox_file}.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") - os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) - # Move private archive mbox there if it's around - # and take into account all sorts of absurdities - print _('- updating old private mbox file') - if os.path.exists(o_pri_mbox_file): - if os.path.isfile(o_pri_mbox_file): - os.rename(o_pri_mbox_file, mbox_file) - elif not os.path.isdir(o_pri_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pri_mbox_file - os.rename(o_pri_mbox_file, newname) - print _("""\ - unknown file in the way, moving - $o_pri_mbox_file - to - $newname""") - else: - # directory - print _("""\ - looks like you have a really recent development installation... - you're either one brave soul, or you already ran me""") - # Move public archive mbox there if it's around - # and take into account all sorts of absurdities. - print _('- updating old public mbox file') - if os.path.exists(o_pub_mbox_file): - if os.path.isfile(o_pub_mbox_file): - os.rename(o_pub_mbox_file, mbox_file) - elif not os.path.isdir(o_pub_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pub_mbox_file - os.rename(o_pub_mbox_file, newname) - print _("""\ - unknown file in the way, moving - $o_pub_mbox_file - to - $newname""") - else: # directory - print _("""\ - looks like you have a really recent development installation... - you're either one brave soul, or you already ran me""") - # Move the html archives there - if os.path.isdir(o_html_dir): - os.rename(o_html_dir, html_dir) - # chmod the html archives - os.chmod(html_dir, 02775) - # BAW: Is this still necessary?! - mlist.Save() - # Check to see if pre-b4 list-specific templates are around - # and move them to the new place if there's not already - # a new one there - tmpl_dir = os.path.join(config.PREFIX, "templates") - list_dir = os.path.join(config.PREFIX, "lists") - b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) - new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) - if os.path.exists(b4_tmpl_dir): - print _("""\ -- This list looks like it might have <= b4 list templates around""") - for f in os.listdir(b4_tmpl_dir): - o_tmpl = os.path.join(b4_tmpl_dir, f) - n_tmpl = os.path.join(new_tmpl_dir, f) - if os.path.exists(o_tmpl): - if not os.path.exists(n_tmpl): - os.rename(o_tmpl, n_tmpl) - print _('- moved $o_tmpl to $n_tmpl') - else: - print _("""\ -- both $o_tmpl and $n_tmpl exist, leaving untouched""") - else: - print _("""\ -- $o_tmpl doesn't exist, leaving untouched""") - # Move all the templates to the en language subdirectory as required for - # Mailman 2.1 - move_language_templates(mlist) - # Avoid eating filehandles with the list lockfiles - mlist.Unlock() - return 0 - - - -def archive_path_fixer(unused_arg, dir, files): - # Passed to os.path.walk to fix the perms on old html archives. - for f in files: - abs = os.path.join(dir, f) - if os.path.isdir(abs): - if f == "database": - os.chmod(abs, 02770) - else: - os.chmod(abs, 02775) - elif os.path.isfile(abs): - os.chmod(abs, 0664) - - -def remove_old_sources(module): - # Also removes old directories. - src = '%s/%s' % (config.PREFIX, module) - pyc = src + "c" - if os.path.isdir(src): - print _('removing directory $src and everything underneath') - shutil.rmtree(src) - elif os.path.exists(src): - print _('removing $src') - try: - os.unlink(src) - except os.error, rest: - print _("Warning: couldn't remove $src -- $rest") - if module.endswith('.py') and os.path.exists(pyc): - try: - os.unlink(pyc) - except OSError, rest: - print _("couldn't remove old file $pyc -- $rest") - - - -def update_qfiles(): - print _('updating old qfiles') - prefix = `time.time()` + '+' - # Be sure the qfiles/in directory exists (we don't really need the - # switchboard object, but it's convenient for creating the directory). - sb = Switchboard(config.INQUEUE_DIR) - for filename in os.listdir(config.QUEUE_DIR): - # Updating means just moving the .db and .msg files to qfiles/in where - # it should be dequeued, converted, and processed normally. - if os.path.splitext(filename) == '.msg': - oldmsgfile = os.path.join(config.QUEUE_DIR, filename) - newmsgfile = os.path.join(config.INQUEUE_DIR, prefix + filename) - os.rename(oldmsgfile, newmsgfile) - elif os.path.splitext(filename) == '.db': - olddbfile = os.path.join(config.QUEUE_DIR, filename) - newdbfile = os.path.join(config.INQUEUE_DIR, prefix + filename) - os.rename(olddbfile, newdbfile) - # Now update for the Mailman 2.1.5 qfile format. For every filebase in - # the qfiles/* directories that has both a .pck and a .db file, pull the - # data out and re-queue them. - for dirname in os.listdir(config.QUEUE_DIR): - dirpath = os.path.join(config.QUEUE_DIR, dirname) - if dirpath == config.BADQUEUE_DIR: - # The files in qfiles/bad can't possibly be pickles - continue - sb = Switchboard(dirpath) - try: - for filename in os.listdir(dirpath): - filepath = os.path.join(dirpath, filename) - filebase, ext = os.path.splitext(filepath) - # Handle the .db metadata files as part of the handling of the - # .pck or .msg message files. - if ext not in ('.pck', '.msg'): - continue - msg, data = dequeue(filebase) - if msg is not None and data is not None: - sb.enqueue(msg, data) - except EnvironmentError, e: - if e.errno <> errno.ENOTDIR: - raise - print _('Warning! Not a directory: $dirpath') - - - -# Implementations taken from the pre-2.1.5 Switchboard -def ext_read(filename): - fp = open(filename) - d = marshal.load(fp) - # Update from version 2 files - if d.get('version', 0) == 2: - del d['filebase'] - # Do the reverse conversion (repr -> float) - for attr in ['received_time']: - try: - sval = d[attr] - except KeyError: - pass - else: - # Do a safe eval by setting up a restricted execution - # environment. This may not be strictly necessary since we - # know they are floats, but it can't hurt. - d[attr] = eval(sval, {'__builtins__': {}}) - fp.close() - return d - - -def dequeue(filebase): - # Calculate the .db and .msg filenames from the given filebase. - msgfile = os.path.join(filebase + '.msg') - pckfile = os.path.join(filebase + '.pck') - dbfile = os.path.join(filebase + '.db') - # Now we are going to read the message and metadata for the given - # filebase. We want to read things in this order: first, the metadata - # file to find out whether the message is stored as a pickle or as - # plain text. Second, the actual message file. However, we want to - # first unlink the message file and then the .db file, because the - # qrunner only cues off of the .db file - msg = None - try: - data = ext_read(dbfile) - os.unlink(dbfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: - raise - data = {} - # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata - # was renamed to `rejection_notice', since dashes in the keys are not - # supported in METAFMT_ASCII. - if data.has_key('rejection-notice'): - data['rejection_notice'] = data['rejection-notice'] - del data['rejection-notice'] - msgfp = None - try: - try: - msgfp = open(pckfile) - msg = cPickle.load(msgfp) - os.unlink(pckfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - msgfp = None - try: - msgfp = open(msgfile) - msg = email.message_from_file(msgfp, Message.Message) - os.unlink(msgfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - except (email.Errors.MessageParseError, ValueError), e: - # This message was unparsable, most likely because its - # MIME encapsulation was broken. For now, there's not - # much we can do about it. - print _('message is unparsable: $filebase') - msgfp.close() - msgfp = None - if config.QRUNNER_SAVE_BAD_MESSAGES: - # Cheapo way to ensure the directory exists w/ the - # proper permissions. - sb = Switchboard(config.BADQUEUE_DIR) - os.rename(msgfile, os.path.join( - config.BADQUEUE_DIR, filebase + '.txt')) - else: - os.unlink(msgfile) - msg = data = None - except EOFError: - # For some reason the pckfile was empty. Just delete it. - print _('Warning! Deleting empty .pck file: $pckfile') - os.unlink(pckfile) - finally: - if msgfp: - msgfp.close() - return msg, data - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - # calculate the versions - lastversion, thisversion = calcversions() - hexlversion = hex(lastversion) - hextversion = hex(thisversion) - if lastversion == thisversion and not opts.force: - # nothing to do - print _('No updates are necessary.') - sys.exit(0) - if lastversion > thisversion and not opts.force: - print _("""\ -Downgrade detected, from version $hexlversion to version $hextversion -This is probably not safe. -Exiting.""") - sys.exit(1) - print _('Upgrading from version $hexlversion to $hextversion') - errors = 0 - # get rid of old stuff - print _('getting rid of old source files') - for mod in ('mailman/Archiver.py', 'mailman/HyperArch.py', - 'mailman/HyperDatabase.py', 'mailman/pipermail.py', - 'mailman/smtplib.py', 'mailman/Cookie.py', - 'bin/update_to_10b6', 'scripts/mailcmd', - 'scripts/mailowner', 'mail/wrapper', 'mailman/pythonlib', - 'cgi-bin/archives', 'mailman/MailCommandHandler'): - remove_old_sources(mod) - if not config.list_manager.names: - print _('no lists == nothing to do, exiting') - return - # For people with web archiving, make sure the directories - # in the archiving are set with proper perms for b6. - if os.path.isdir("%s/public_html/archives" % config.PREFIX): - print _("""\ -fixing all the perms on your old html archives to work with b6 -If your archives are big, this could take a minute or two...""") - os.path.walk("%s/public_html/archives" % config.PREFIX, - archive_path_fixer, "") - print _('done') - for listname in config.list_manager.names: - # With 2.2.0a0, all list names grew an @domain suffix. If you find a - # list without that, move it now. - if not '@' in listname: - listname = situate_list(listname) - print _('Updating mailing list: $listname') - errors += dolist(listname) - print - print _('Updating Usenet watermarks') - wmfile = os.path.join(config.DATA_DIR, 'gate_watermarks') - try: - fp = open(wmfile) - except IOError: - print _('- nothing to update here') - else: - d = marshal.load(fp) - fp.close() - for listname in d.keys(): - if listname not in listnames: - # this list no longer exists - continue - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: $listname') - errors = errors + 1 - else: - # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate - # that no gating had been done yet. Without coercing this to - # None, the list could now suddenly get flooded. - mlist.usenet_watermark = d[listname] or None - mlist.Save() - mlist.Unlock() - os.unlink(wmfile) - print _('- usenet watermarks updated and gate_watermarks removed') - # In Mailman 2.1, the qfiles directory has a different structure and a - # different content. Also, in Mailman 2.1.5 we collapsed the message - # files from separate .msg (pickled Message objects) and .db (marshalled - # dictionaries) to a shared .pck file containing two pickles. - update_qfiles() - # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. - # There's no good way of figuring this out for releases prior to 2.0beta2 - # :( - if lastversion == NOTFRESH: - print _(""" - -NOTE NOTE NOTE NOTE NOTE - - You are upgrading an existing Mailman installation, but I can't tell what - version you were previously running. - - If you are upgrading from Mailman 1.0b9 or earlier you will need to - manually update your mailing lists. For each mailing list you need to - copy the file templates/options.html lists/<listname>/options.html. - - However, if you have edited this file via the Web interface, you will have - to merge your changes into this file, otherwise you will lose your - changes. - -NOTE NOTE NOTE NOTE NOTE - -""") - if not errors: - # Record the version we just upgraded to - fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version'), 'w') - fp.write(hex(config.HEX_VERSION) + '\n') - fp.close() - else: - lockdir = config.LOCK_DIR - print _('''\ - -ERROR: - -The locks for some lists could not be acquired. This means that either -Mailman was still active when you upgraded, or there were stale locks in the -$lockdir directory. - -You must put Mailman into a quiescent state and remove all stale locks, then -re-run "make update" manually. See the INSTALL and UPGRADE files for details. -''') diff --git a/src/mailman/chains/accept.py b/src/mailman/chains/accept.py index b1e4b1cf0..4b326142b 100644 --- a/src/mailman/chains/accept.py +++ b/src/mailman/chains/accept.py @@ -62,7 +62,6 @@ class AcceptChain(TerminalChainBase): rule_misses = msgdata.get('rule_misses') if rule_misses: msg['X-Mailman-Rule-Misses'] = SEMISPACE.join(rule_misses) - accept_queue = config.switchboards['pipeline'] - accept_queue.enqueue(msg, msgdata) + config.switchboards['pipeline'].enqueue(msg, msgdata) log.info('ACCEPT: %s', msg.get('message-id', 'n/a')) notify(AcceptNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/owner.py b/src/mailman/chains/owner.py new file mode 100644 index 000000000..ad0a04cea --- /dev/null +++ b/src/mailman/chains/owner.py @@ -0,0 +1,56 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""The standard -owner posting chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BuiltInOwnerChain', + ] + + +import logging + +from zope.event import notify + +from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.config import config +from mailman.core.i18n import _ + + +log = logging.getLogger('mailman.vette') + + + +class OwnerNotification(ChainNotification): + """An event signaling that a message is accepted to the -owner address.""" + + + +class BuiltInOwnerChain(TerminalChainBase): + """Default built-in -owner address chain.""" + + name = 'default-owner-chain' + description = _('The built-in -owner posting chain.') + + def _process(self, mlist, msg, msgdata): + # At least for now, everything posted to -owners goes through. + config.switchboards['pipeline'].enqueue(msg, msgdata) + log.info('OWNER: %s', msg.get('message-id', 'n/a')) + notify(OwnerNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/tests/test_owner.py b/src/mailman/chains/tests/test_owner.py new file mode 100644 index 000000000..db85d4967 --- /dev/null +++ b/src/mailman/chains/tests/test_owner.py @@ -0,0 +1,75 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the owner chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestOwnerChain', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.owner import BuiltInOwnerChain, OwnerNotification +from mailman.core.chains import process +from mailman.testing.helpers import ( + event_subscribers, + get_queue_messages, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestOwnerChain(unittest.TestCase): + """Test the owner chain.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Message-ID: <ant> + +""") + + def test_owner_pipeline(self): + # Messages processed through the default owners chain end up in the + # pipeline queue, and an event gets sent. + # + # This event subscriber records the event that occurs when the message + # is processed by the owner chain. + events = [] + def catch_event(event): + events.append(event) + with event_subscribers(catch_event): + process(self._mlist, self._msg, {}, 'default-owner-chain') + self.assertEqual(len(events), 1) + event = events[0] + self.assertTrue(isinstance(event, OwnerNotification)) + self.assertEqual(event.mlist, self._mlist) + self.assertEqual(event.msg['message-id'], '<ant>') + self.assertTrue(isinstance(event.chain, BuiltInOwnerChain)) + messages = get_queue_messages('pipeline') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message['message-id'], '<ant>') diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index 42e67e3a8..af6afe22d 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -109,7 +109,7 @@ class Lists: for mlist in mailing_lists: if args.names: identifier = '{0} [{1}]'.format( - mlist.fqdn_listname, mlist.real_name) + mlist.fqdn_listname, mlist.display_name) else: identifier = mlist.fqdn_listname longest = max(len(identifier), longest) @@ -252,12 +252,6 @@ class Remove: def add(self, parser, command_parser): """See `ICLISubCommand`.""" command_parser.add_argument( - '-a', '--archives', - default=False, action='store_true', - help=_("""\ -Remove the list's archives too, or if the list has already been deleted, -remove any residual archives.""")) - command_parser.add_argument( '-q', '--quiet', default=False, action='store_true', help=_('Suppress status messages')) @@ -278,15 +272,9 @@ remove any residual archives.""")) fqdn_listname = args.listname[0] mlist = getUtility(IListManager).get(fqdn_listname) if mlist is None: - if args.archives: - log(_('No such list: $fqdn_listname; ' - 'removing residual archives.')) - else: - log(_('No such list: $fqdn_listname')) - return + log(_('No such list: $fqdn_listname')) + return else: log(_('Removed list: $fqdn_listname')) - if not args.archives: - log(_('Not removing archives. Reinvoke with -a to remove them.')) - remove_list(fqdn_listname, mlist, args.archives) + remove_list(fqdn_listname, mlist) config.db.commit() diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index f37fb6ecb..2bf6be848 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -17,7 +17,7 @@ """The 'members' subcommand.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -155,7 +155,7 @@ class Members: try: addresses = list(mlist.members.addresses) if len(addresses) == 0: - print >> fp, mlist.fqdn_listname, 'has no members' + print(mlist.fqdn_listname, 'has no members', file=fp) return for address in sorted(addresses, key=attrgetter('email')): if args.regular: @@ -170,8 +170,9 @@ class Members: member = mlist.members.get_member(address.email) if member.delivery_status not in status_types: continue - print >> fp, formataddr( - (address.real_name, address.original_email)) + print( + formataddr((address.display_name, address.original_email)), + file=fp) finally: if fp is not sys.stdout: fp.close() @@ -194,19 +195,20 @@ class Members: if line.startswith('#') or len(line.strip()) == 0: continue # Parse the line and ensure that the values are unicodes. - real_name, email = parseaddr(line) - real_name = real_name.decode(fp.encoding) + display_name, email = parseaddr(line) + display_name = display_name.decode(fp.encoding) email = email.decode(fp.encoding) # Give the user a default, user-friendly password. password = generate(int(config.passwords.password_length)) try: - add_member(mlist, email, real_name, password, + add_member(mlist, email, display_name, password, DeliveryMode.regular, mlist.preferred_language.code) except AlreadySubscribedError: # It's okay if the address is already subscribed, just # print a warning and continue. - print 'Already subscribed (skipping):', email, real_name + print('Already subscribed (skipping):', + email, display_name) finally: if fp is not sys.stdin: fp.close() diff --git a/src/mailman/commands/cli_withlist.py b/src/mailman/commands/cli_withlist.py index 3b1b36b1a..55d9ff5ec 100644 --- a/src/mailman/commands/cli_withlist.py +++ b/src/mailman/commands/cli_withlist.py @@ -230,8 +230,8 @@ As another example, say you wanted to change the display name for a particular mailing list. You could put the following function in a file called 'change.pw': - def change(mlist, real_name): - mlist.real_name = real_name + def change(mlist, display_name): + mlist.display_name = display_name # Required to save changes to the database. commit() diff --git a/src/mailman/commands/docs/import.rst b/src/mailman/commands/docs/import.rst index 1092063a4..2ab6f99bd 100644 --- a/src/mailman/commands/docs/import.rst +++ b/src/mailman/commands/docs/import.rst @@ -48,9 +48,9 @@ import, the mailing list's 'real name' has changed. >>> FakeArgs.pickle_file = [ ... resource_filename('mailman.testing', 'config.pck')] - >>> print mlist.real_name + >>> print mlist.display_name Import >>> command.process(FakeArgs) - >>> print mlist.real_name + >>> print mlist.display_name Test diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 34883711e..7f69eada5 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -59,6 +59,7 @@ The File System Hierarchy layout is the same every by definition. Python ... ... File system paths: + ARCHIVE_DIR = /var/lib/mailman/archives BIN_DIR = /sbin DATA_DIR = /var/lib/mailman/data ETC_DIR = /etc @@ -69,8 +70,6 @@ The File System Hierarchy layout is the same every by definition. LOG_DIR = /var/log/mailman MESSAGES_DIR = /var/lib/mailman/messages PID_FILE = /var/run/mailman/master.pid - PRIVATE_ARCHIVE_FILE_DIR = /var/lib/mailman/archives/private - PUBLIC_ARCHIVE_FILE_DIR = /var/lib/mailman/archives/public QUEUE_DIR = /var/spool/mailman TEMPLATE_DIR = .../mailman/templates VAR_DIR = /var/lib/mailman diff --git a/src/mailman/commands/docs/membership.rst b/src/mailman/commands/docs/membership.rst index f9d3aacd0..3faccfe6a 100644 --- a/src/mailman/commands/docs/membership.rst +++ b/src/mailman/commands/docs/membership.rst @@ -157,7 +157,7 @@ list. <BLANKLINE> >>> user = user_manager.get_user('anne@example.com') - >>> print user.real_name + >>> print user.display_name Anne Person >>> list(user.addresses) [<Address: Anne Person <anne@example.com> [verified] at ...>] @@ -292,8 +292,8 @@ Once Anne has verified her alternative address though, it can be used to unsubscribe her from the list. :: - >>> from datetime import datetime - >>> address.verified_on = datetime.now() + >>> from mailman.utilities.datetime import now + >>> address.verified_on = now() >>> results = Results() >>> print leave.process(mlist, msg, {}, (), results) diff --git a/src/mailman/commands/docs/qfile.rst b/src/mailman/commands/docs/qfile.rst index 74ede1b64..8ec0a3952 100644 --- a/src/mailman/commands/docs/qfile.rst +++ b/src/mailman/commands/docs/qfile.rst @@ -59,7 +59,6 @@ Once we've figured out the file name of the shunted message, we can print it. 'bad': u'yes', 'bar': u'baz', 'foo': 7, - u'received_time': ... u'version': 3} [----- end pickle -----] diff --git a/src/mailman/commands/docs/remove.rst b/src/mailman/commands/docs/remove.rst index f0f4e64f6..35dc53c5e 100644 --- a/src/mailman/commands/docs/remove.rst +++ b/src/mailman/commands/docs/remove.rst @@ -24,7 +24,6 @@ A system administrator can remove mailing lists by the command line. >>> command = Remove() >>> command.process(args) Removed list: test@example.com - Not removing archives. Reinvoke with -a to remove them. >>> print list_manager.get('test@example.com') None @@ -40,46 +39,3 @@ You can also remove lists quietly. >>> print list_manager.get('test@example.com') None - - -Removing archives -================= - -By default 'mailman remove' does not remove a mailing list's archives. -:: - - >>> create_list('test@example.com') - <mailing list "test@example.com" at ...> - - # Fake an mbox file for the mailing list. - >>> import os - >>> def make_mbox(fqdn_listname): - ... mbox_dir = os.path.join( - ... config.PUBLIC_ARCHIVE_FILE_DIR, fqdn_listname + '.mbox') - ... os.makedirs(mbox_dir) - ... mbox_file = os.path.join(mbox_dir, fqdn_listname + '.mbox') - ... with open(mbox_file, 'w') as fp: - ... print >> fp, 'A message' - ... assert os.path.exists(mbox_file) - ... return mbox_file - - >>> mbox_file = make_mbox('test@example.com') - >>> args.quiet = False - >>> command.process(args) - Removed list: test@example.com - Not removing archives. Reinvoke with -a to remove them. - - >>> os.path.exists(mbox_file) - True - -Even if the mailing list has been deleted, you can still delete the archives -afterward. -:: - - >>> args.archives = True - - >>> command.process(args) - No such list: test@example.com; removing residual archives. - - >>> os.path.exists(mbox_file) - False diff --git a/src/mailman/commands/docs/withlist.rst b/src/mailman/commands/docs/withlist.rst index f00208490..99a366c9a 100644 --- a/src/mailman/commands/docs/withlist.rst +++ b/src/mailman/commands/docs/withlist.rst @@ -54,8 +54,8 @@ single argument, the mailing list. ... def showme(mailing_list): ... print "The list's name is", mailing_list.fqdn_listname ... - ... def realname(mailing_list): - ... print "The list's real name is", mailing_list.real_name + ... def displayname(mailing_list): + ... print "The list's display name is", mailing_list.display_name ... """ If the name of the function is the same as the module, then you only need to @@ -71,9 +71,9 @@ name the function once. The function's name can also be different than the modules name. In that case, just give the full module path name to the function you want to call. - >>> args.run = 'showme.realname' + >>> args.run = 'showme.displayname' >>> command.process(args) - The list's real name is Aardvark + The list's display name is Aardvark Multiple lists @@ -89,14 +89,14 @@ must start with a caret. >>> args.listname = '^.*example.com' >>> command.process(args) - The list's real name is Aardvark - The list's real name is Badger - The list's real name is Badboys + The list's display name is Aardvark + The list's display name is Badger + The list's display name is Badboys >>> args.listname = '^bad.*' >>> command.process(args) - The list's real name is Badger - The list's real name is Badboys + The list's display name is Badger + The list's display name is Badboys >>> args.listname = '^foo' >>> command.process(args) diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py index 386316eb9..d6f7a47d9 100644 --- a/src/mailman/commands/eml_membership.py +++ b/src/mailman/commands/eml_membership.py @@ -65,7 +65,7 @@ used. delivery_mode = self._parse_arguments(arguments, results) if delivery_mode is ContinueProcessing.no: return ContinueProcessing.no - real_name, address = parseaddr(msg['from']) + display_name, address = parseaddr(msg['from']) # Address could be None or the empty string. if not address: address = msg.sender @@ -81,7 +81,7 @@ used. return ContinueProcessing.yes joins.add(address) results.joins = joins - person = formataddr((real_name, address)) + person = formataddr((display_name, address)) # Is this person already a member of the list? Search for all # matching memberships. members = getUtility(ISubscriptionService).find_members( @@ -90,7 +90,7 @@ used. print(_('$person is already a member'), file=results) else: getUtility(IRegistrar).register(mlist, address, - real_name, delivery_mode) + display_name, delivery_mode) print(_('Confirmation email sent to $person'), file=results) return ContinueProcessing.yes @@ -177,7 +177,7 @@ You may be asked to confirm your request.""") file=results) return ContinueProcessing.no member.unsubscribe() - person = formataddr((user.real_name, email)) + person = formataddr((user.display_name, email)) print(_('$person left $mlist.fqdn_listname'), file=results) return ContinueProcessing.yes diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 034b76b4f..47ef021b5 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -173,8 +173,7 @@ class Configuration: lock_dir = category.lock_dir, log_dir = category.log_dir, messages_dir = category.messages_dir, - pipermail_private_dir = category.pipermail_private_dir, - pipermail_public_dir = category.pipermail_public_dir, + archive_dir = category.archive_dir, queue_dir = category.queue_dir, var_dir = var_dir, template_dir = ( @@ -208,10 +207,6 @@ class Configuration: # Ensure that all paths are normalized and made absolute. Handle the # few special cases first. Most of these are due to backward # compatibility. - self.PUBLIC_ARCHIVE_FILE_DIR = os.path.abspath( - substitutions.pop('pipermail_public_dir')) - self.PRIVATE_ARCHIVE_FILE_DIR = os.path.abspath( - substitutions.pop('pipermail_private_dir')) self.PID_FILE = os.path.abspath(substitutions.pop('pid_file')) for key in substitutions: attribute = key.upper() diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index e662633e6..88f378159 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -62,6 +62,10 @@ post_hook: # Which paths.* file system layout to use. layout: dev +# Can MIME filtered messages be preserved by list owners? +filtered_messages_are_preservable: no + + [shell] # `bin/mailman shell` (also `withlist`) gives you an interactive prompt that # you can use to interact with an initialized and configured Mailman system. @@ -113,10 +117,9 @@ etc_dir: $var_dir/etc ext_dir: $var_dir/ext # Directory where the default IMessageStore puts its messages. messages_dir: $var_dir/messages -# Directory for public Pipermail archiver artifacts. -pipermail_public_dir: $var_dir/archives/public -# Directory for private Pipermail archiver artifacts. -pipermail_private_dir: $var_dir/archives/private +# Directory for archive backends to store their messages in. Archivers should +# create a subdirectory in here to store their files. +archive_dir: $var_dir/archives # Root directory for site-specific template override files. template_dir: $var_dir/templates # There are also a number of paths to specific file locations that can be @@ -230,12 +233,11 @@ migrations_path: mailman.database.schema # - http -- Internal wsgi-based web interface # - locks -- Lock state changes # - mischief -- Various types of hostile activity -# - post -- Information about messages posted to mailing lists # - runner -- Runner process start/stops # - smtp -- Successful SMTP activity # - smtp-failure -- Unsuccessful SMTP activity # - subscribe -- Information about leaves/joins -# - vette -- Information related to admindb activity +# - vette -- Message vetting information format: %(asctime)s (%(process)d) %(message)s datefmt: %b %d %H:%M:%S %Y propagate: no @@ -538,6 +540,22 @@ recipient: archive@archive.example.com # command to call. command: /bin/echo +# When sending the message to the archiver, you have the option of +# "clobbering" the Date: header, specifically to make it more sane. Some +# archivers can't handle dates that are wildly off from reality. This does +# not change the Date: header for any other delivery vector except this +# specific archive. +# +# When the original Date header is clobbered, it will always be stored in +# X-Original-Date. The new Date header will always be set to the date at +# which the messages was received by the Mailman server, in UTC. +# +# Your options here are: +# * never -- Leaves the original Date header alone. +# * always -- Always override the Date header. +# * maybe -- Override the Date only if it is outside the clobber_skew period. +clobber_date: maybe +clobber_skew: 1d [archiver.mhonarc] # This is the stock MHonArc archiver. @@ -550,32 +568,6 @@ base_url: http://$hostname/archives/$fqdn_listname # This is the stock mail-archive.com archiver. class: mailman.archiving.mailarchive.MailArchive -[archiver.pipermail] -# This is the stock Pipermail archiver. -class: mailman.archiving.pipermail.Pipermail - -# This sets the default `clobber date' policy for the archiver. When a -# message is to be archived either by Pipermail or an external archiver, -# Mailman can modify the Date: header to be the date the message was received -# instead of the Date: in the original message. This is useful if you -# typically receive messages with outrageous dates. Set this to 0 to retain -# the date of the original message, or to 1 to always clobber the date. Set -# it to 2 to perform `smart overrides' on the date; when the date is outside -# allowable_sane_date_skew (either too early or too late), then the received -# date is substituted instead. -clobber_date_policy: 2 -allowable_sane_date_skew: 15d - -# Pipermail archives contain the raw email addresses of the posting authors. -# Some view this as a goldmine for spam harvesters. Set this to 'yes' to -# moderately obscure email addresses, but note that this breaks mailto: URLs -# in the archives too. -obscure_email_addresses: yes - -# When the archive is public, should Pipermail also make the raw Unix mbox -# file publically available? -public_mbox: no - [archiver.prototype] # This is a prototypical sample archiver. @@ -590,59 +582,6 @@ priority: 0 class: mailman.styles.default.DefaultStyle -[scrubber] -# A filter that converts from multipart messages to "flat" messages -# (i.e. containing a single payload). This is required for Pipermail, and you -# may want to set it to 0 for external archivers. You can also replace it -# with your own module as long as it contains a process() function that takes -# a MailList object and a Message object. It should raise -# Errors.DiscardMessage if it wants to throw the message away. Otherwise it -# should modify the Message object as necessary. -archive_scrubber: mailman.archiving.pipermail.Pipermail - -# This variable defines what happens to text/html subparts. They can be -# stripped completely, escaped, or filtered through an external program. The -# legal values are: -# 0 - Strip out text/html parts completely, leaving a notice of the removal in -# the message. If the outer part is text/html, the entire message is -# discarded. -# 1 - Remove any embedded text/html parts, leaving them as HTML-escaped -# attachments which can be separately viewed. Outer text/html parts are -# simply HTML-escaped. -# 2 - Leave it inline, but HTML-escape it -# 3 - Remove text/html as attachments but don't HTML-escape them. Note: this -# is very dangerous because it essentially means anybody can send an HTML -# email to your site containing evil JavaScript or web bugs, or other -# nasty things, and folks viewing your archives will be susceptible. You -# should only consider this option if you do heavy moderation of your list -# postings. -# -# Note: given the current archiving code, it is not possible to leave -# text/html parts inline and un-escaped. I wouldn't think it'd be a good idea -# to do anyway. -# -# The value can also be a string, in which case it is the name of a command to -# filter the HTML page through. The resulting output is left in an attachment -# or as the entirety of the message when the outer part is text/html. The -# format of the string must include a $filename substitution variable which -# will contain the name of the temporary file that the program should operate -# on. It should write the processed message to stdout. Set this to -# HTML_TO_PLAIN_TEXT_COMMAND to specify an HTML to plain text conversion -# program. -archive_html_sanitizer: 1 - -# Control parameter whether the scrubber should use the message attachment's -# filename as is indicated by the filename parameter or use 'attachement-xxx' -# instead. The default is set 'no' because the applications on PC and Mac -# begin to use longer non-ascii filenames. -use_attachment_filename: no - -# Use of attachment filename extension per se is may be dangerous because -# viruses fakes it. You can set this 'yes' if you filter the attachment by -# filename extension. -use_attachment_filename_extension: no - - [digests] # Headers which should be kept in both RFC 1153 (plain) and MIME digests. RFC # 1153 also specifies these headers in this exact order, so order matters. diff --git a/src/mailman/core/errors.py b/src/mailman/core/errors.py index ea1c78967..529ac86fe 100644 --- a/src/mailman/core/errors.py +++ b/src/mailman/core/errors.py @@ -110,17 +110,6 @@ class DiscardMessage(HandlerError): class RejectMessage(HandlerError): """The message will be bounced back to the sender""" - def __init__(self, notice=None): - super(RejectMessage, self).__init__() - if notice is None: - notice = _('Your message was rejected') - if notice.endswith('\n\n'): - pass - elif notice.endswith('\n'): - notice += '\n' - else: - notice += '\n\n' - self.notice = notice diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 721877056..389a45f3b 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -108,9 +108,7 @@ def initialize_1(config_path=None): # By default, set the umask so that only owner and group can read and # write our files. Specifically we must have g+rw and we probably want # o-rwx although I think in most cases it doesn't hurt if other can read - # or write the files. Note that the Pipermail archive has more - # restrictive permissions in order to handle private archives, but it - # handles that correctly. + # or write the files. os.umask(007) # config_path will be set if the command line argument -C is given. That # case overrides all others. When not given on the command line, the diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index bd709f41e..25bb68030 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -15,12 +15,16 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Pipeline processor.""" +"""Built-in pipelines.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'BasePipeline', + 'OwnerPipeline', + 'PostingPipeline', + 'VirginPipeline', 'initialize', 'process', ] @@ -31,13 +35,16 @@ import logging from zope.interface import implements from zope.interface.verify import verifyObject +from mailman.app.bounces import bounce_message from mailman.app.finder import find_components from mailman.config import config +from mailman.core import errors from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.pipeline import IPipeline -log = logging.getLogger('mailman.debug') +dlog = logging.getLogger('mailman.debug') +vlog = logging.getLogger('mailman.vette') @@ -52,9 +59,19 @@ def process(mlist, msg, msgdata, pipeline_name='built-in'): message_id = msg.get('message-id', 'n/a') pipeline = config.pipelines[pipeline_name] for handler in pipeline: - log.debug('[pipeline] processing {0}: {1}'.format( - handler.name, message_id)) - handler.process(mlist, msg, msgdata) + dlog.debug('{0} pipeline {1} processing: {2}'.format( + message_id, pipeline_name, handler.name)) + try: + handler.process(mlist, msg, msgdata) + except errors.DiscardMessage as error: + vlog.info( + '{0} discarded by "{1}" pipeline handler "{2}": {3}'.format( + message_id, pipeline_name, handler.name, error.message)) + except errors.RejectMessage as error: + vlog.info( + '{0} rejected by "{1}" pipeline handler "{2}": {3}'.format( + message_id, pipeline_name, handler.name, error.message)) + bounce_message(mlist, msg, error) @@ -76,24 +93,36 @@ class BasePipeline: yield handler -class BuiltInPipeline(BasePipeline): - """The built-in pipeline.""" + +class OwnerPipeline(BasePipeline): + """The built-in owner pipeline.""" + + name = 'default-owner-pipeline' + description = _('The built-in owner pipeline.') + + _default_handlers = ( + 'owner-recipients', + 'to-outgoing', + ) + + +class PostingPipeline(BasePipeline): + """The built-in posting pipeline.""" name = 'default-posting-pipeline' - description = _('The built-in pipeline.') + description = _('The built-in posting pipeline.') _default_handlers = ( 'mime-delete', - 'scrubber', 'tagger', - 'calculate-recipients', + 'member-recipients', 'avoid-duplicates', 'cleanse', 'cleanse-dkim', 'cook-headers', 'rfc-2369', - 'to-digest', 'to-archive', + 'to-digest', 'to-usenet', 'after-delivery', 'acknowledge', @@ -119,7 +148,7 @@ class VirginPipeline(BasePipeline): def initialize(): """Initialize the pipelines.""" # Find all handlers in the registered plugins. - for handler_class in find_components('mailman.pipeline', IHandler): + for handler_class in find_components('mailman.handlers', IHandler): handler = handler_class() verifyObject(IHandler, handler) assert handler.name not in config.handlers, ( @@ -127,6 +156,6 @@ def initialize(): handler.name, handler_class)) config.handlers[handler.name] = handler # Set up some pipelines. - for pipeline_class in (BuiltInPipeline, VirginPipeline): + for pipeline_class in (OwnerPipeline, PostingPipeline, VirginPipeline): pipeline = pipeline_class() config.pipelines[pipeline.name] = pipeline diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index ba8b2ec3f..7cab4f4ad 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -137,8 +137,7 @@ class Switchboard: # file name consists of two parts separated by a '+': the received # time for this message (i.e. when it first showed up on this system) # and the sha hex digest. - rcvtime = data.setdefault('received_time', now) - filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() + filebase = repr(now) + '+' + hashlib.sha1(hashfood).hexdigest() filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number diff --git a/src/mailman/core/tests/test_pipelines.py b/src/mailman/core/tests/test_pipelines.py index 363587d3b..8f851de95 100644 --- a/src/mailman/core/tests/test_pipelines.py +++ b/src/mailman/core/tests/test_pipelines.py @@ -21,43 +21,159 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestOwnerPipeline', + 'TestPostingPipeline', ] import unittest +from zope.component import getUtility +from zope.interface import implements from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.core.errors import DiscardMessage, RejectMessage from mailman.core.pipelines import process +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import MemberRole +from mailman.interfaces.pipeline import IPipeline +from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( + LogFileMark, + get_queue_messages, reset_the_world, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer -class TestBuiltinPipeline(unittest.TestCase): +class DiscardingHandler: + implements(IHandler) + name = 'discarding' + + def process(self, mlist, msg, msgdata): + raise DiscardMessage('by test handler') + + +class RejectHandler: + implements(IHandler) + name = 'rejecting' + + def process(self, mlist, msg, msgdata): + raise RejectMessage('by test handler') + + +class DiscardingPipeline: + implements(IPipeline) + name = 'test-discarding' + description = 'Discarding test pipeline' + + def __iter__(self): + yield DiscardingHandler() + + +class RejectingPipeline: + implements(IPipeline) + name = 'test-rejecting' + description = 'Rejectinging test pipeline' + + def __iter__(self): + yield RejectHandler() + + + +class TestPostingPipeline(unittest.TestCase): """Test various aspects of the built-in postings pipeline.""" layer = ConfigLayer def setUp(self): self._mlist = create_list('test@example.com') + config.pipelines['test-discarding'] = DiscardingPipeline() + config.pipelines['test-rejecting'] = RejectingPipeline() + self._msg = mfs("""\ +From: Anne Person <anne@example.org> +To: test@example.com +Subject: a test +Message-ID: <ant> + +testing +""") def tearDown(self): reset_the_world() + del config.pipelines['test-discarding'] + del config.pipelines['test-rejecting'] def test_rfc2369_headers(self): # Ensure that RFC 2369 List-* headers are added. - msg = mfs("""\ + msgdata = {} + process(self._mlist, self._msg, msgdata, + pipeline_name='default-posting-pipeline') + self.assertEqual(self._msg['list-id'], '<test.example.com>') + self.assertEqual(self._msg['list-post'], '<mailto:test@example.com>') + + def test_discarding_pipeline(self): + # If a handler in the pipeline raises DiscardMessage, the message will + # be thrown away, but with a log message. + mark = LogFileMark('mailman.vette') + process(self._mlist, self._msg, {}, 'test-discarding') + line = mark.readline()[:-1] + self.assertTrue(line.endswith( + '<ant> discarded by "test-discarding" pipeline handler ' + '"discarding": by test handler')) + + def test_rejecting_pipeline(self): + # If a handler in the pipeline raises DiscardMessage, the message will + # be thrown away, but with a log message. + mark = LogFileMark('mailman.vette') + process(self._mlist, self._msg, {}, 'test-rejecting') + line = mark.readline()[:-1] + self.assertTrue(line.endswith( + '<ant> rejected by "test-rejecting" pipeline handler ' + '"rejecting": by test handler')) + # In the rejection case, the original message will also be in the + # virgin queue. + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 1) + self.assertEqual(str(messages[0].msg['subject']), 'a test') + + + +class TestOwnerPipeline(unittest.TestCase): + """Test various aspects of the built-in owner pipeline.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + user_manager = getUtility(IUserManager) + anne = user_manager.create_address('anne@example.com') + bart = user_manager.create_address('bart@example.com') + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._msg = mfs("""\ From: Anne Person <anne@example.org> -To: test@example.com -Subject: a test +To: test-owner@example.com -testing """) - msgdata = {} - process(self._mlist, msg, msgdata, - pipeline_name='default-posting-pipeline') - self.assertEqual(msg['list-id'], '<test.example.com>') - self.assertEqual(msg['list-post'], '<mailto:test@example.com>') + + def test_calculate_recipients(self): + # Recipients are the administrators of the mailing list. + msgdata = dict(listname='test@example.com', + to_owner=True) + process(self._mlist, self._msg, msgdata, + pipeline_name='default-owner-pipeline') + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com'))) + + def test_to_outgoing(self): + # The message, with the calculated recipients, gets put in the + # outgoing queue. + process(self._mlist, self._msg, {}, + pipeline_name='default-owner-pipeline') + messages = get_queue_messages('out', sort_on='to') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['recipients'], + set(('anne@example.com', 'bart@example.com'))) diff --git a/src/mailman/database/schema/postgres.sql b/src/mailman/database/schema/postgres.sql index 713d6d1a3..2e9ba249f 100644 --- a/src/mailman/database/schema/postgres.sql +++ b/src/mailman/database/schema/postgres.sql @@ -42,6 +42,7 @@ CREATE TABLE mailinglist ( bounce_you_are_disabled_warnings INTEGER, bounce_you_are_disabled_warnings_interval TEXT, -- Content filtering. + filter_action INTEGER, filter_content BOOLEAN, collapse_alternatives BOOLEAN, convert_html_to_plaintext BOOLEAN, @@ -83,13 +84,15 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, posting_pipeline TEXT, preferred_language TEXT, private_roster BOOLEAN, - real_name TEXT, + display_name TEXT, reject_these_nonmembers BYTEA, reply_goes_to_list INTEGER, reply_to_address TEXT, @@ -154,7 +157,7 @@ CREATE TABLE address ( id SERIAL NOT NULL, email TEXT, _original TEXT, - real_name TEXT, + display_name TEXT, verified_on TIMESTAMP, registered_on TIMESTAMP, user_id INTEGER, @@ -168,7 +171,7 @@ CREATE TABLE address ( CREATE TABLE "user" ( id SERIAL NOT NULL, - real_name TEXT, + display_name TEXT, password BYTEA, _user_id UUID, _created_on TIMESTAMP, diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql index f835a8d84..e6211bf53 100644 --- a/src/mailman/database/schema/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -27,7 +27,7 @@ CREATE TABLE address ( id INTEGER NOT NULL, email TEXT, _original TEXT, - real_name TEXT, + display_name TEXT, verified_on TIMESTAMP, registered_on TIMESTAMP, user_id INTEGER, @@ -138,6 +138,7 @@ CREATE TABLE mailinglist ( bounce_you_are_disabled_warnings INTEGER, bounce_you_are_disabled_warnings_interval TEXT, -- Content filtering. + filter_action INTEGER, filter_content BOOLEAN, collapse_alternatives BOOLEAN, convert_html_to_plaintext BOOLEAN, @@ -179,13 +180,15 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, posting_pipeline TEXT, preferred_language TEXT, private_roster BOOLEAN, - real_name TEXT, + display_name TEXT, reject_these_nonmembers BLOB, reply_goes_to_list INTEGER, reply_to_address TEXT, @@ -278,7 +281,7 @@ CREATE TABLE preferences ( CREATE TABLE user ( id INTEGER NOT NULL, - real_name TEXT, + display_name TEXT, password BINARY, _user_id TEXT, _created_on TIMESTAMP, diff --git a/src/mailman/docs/8-miles-high.rst b/src/mailman/docs/8-miles-high.rst index 0869bd563..812d78d72 100644 --- a/src/mailman/docs/8-miles-high.rst +++ b/src/mailman/docs/8-miles-high.rst @@ -8,26 +8,27 @@ Notes from the PyCon 2012 Mailman Sprint diagrams from his "Mailman" presentation at PyCon 2012. Transcribed by Stephen Turnbull. -These are notes from the Mailman sprint at PyCon 2012. They are not +*These are notes from the Mailman sprint at PyCon 2012. They are not terribly well organized, nor fully fleshed out. Please edit and push -branches to Launchpad at lp:mailman or post patches to <WHERE?> <URL?>. +branches to Launchpad at lp:mailman or post patches to +<https://bugs.launchpad.net/mailman>.* -The intent of this document is to provide a view of Mailman 3's -workflow and structures from "eight miles high". +The intent of this document is to provide a view of Mailman 3's workflow and +structures from "eight miles high". Basic Messaging Handling Workflow ---------------------------------- +================================= -Mailman accepts a message via the LMTP protocol (RFC 2033). It -implements a simple LMTP server internally based on the LMTP server -provided in the Python stdlib. The LMTP server's responsibility is to -parse the message into a tuple (*mlist*, *msg*, *msg_data*). If the -parse fails (including messages which Mailman considers to be invalid -due to lack of Message-Id as strongly recommended by RFC 2822 and RFC -5322), the message will be rejected, otherwise the tuple is pickled, -and the resulting *message pickle* added to one of the IN, COMMAND, or -BOUNCE processing queues. +Mailman accepts a message via the LMTP protocol (RFC 2033). It implements a +simple LMTP server internally based on the LMTP server provided in the Python +standard library. The LMTP server's responsibility is to parse the message +into a tuple (*mlist*, *msg*, *msgdata*). If the parse fails (including +messages which Mailman considers to be invalid due to lack of `Message-Id` as +strongly recommended by RFC 2822 and RFC 5322), the message will be rejected, +otherwise the parsed message and metadata dictionary are pickled, and the +resulting *message pickle* added to one of the `in`, `command`, or `bounce` +processing queues. .. graphviz:: @@ -39,27 +40,27 @@ BOUNCE processing queues. msg -> MTA [label="SMTP"]; MTA -> lmtpd [label="LMTP"]; lmtpd -> MTA [label="reject"]; - lmtpd -> IN -> POSTING [label=".pck"]; + lmtpd -> IN -> PIPELINE [label=".pck"]; lmtpd -> BOUNCES [label=".pck"]; lmtpd -> COMMAND [label=".pck"]; } -The IN queue is processed by *filter chains* (explained below) to -determine whether the post (or administrative request) will be -processed. If not allowed, the message pickle is discarded, rejected -(returned to sender), or held (added to the MODERATION queue -- not -shown). Otherwise the message is added to the POSTING queue. +The `in` queue is processed by *filter chains* (explained below) to determine +whether the post (or administrative request) will be processed. If not +allowed, the message pickle is discarded, rejected (returned to sender), or +held (saved for moderator approval -- not shown). Otherwise the message is +added to the `pipeline` (i.e. posting) queue. -Each of the COMMAND, BOUNCE, and POSTING queues is processed by a -*pipeline of handlers* as in Mailman 2's pipeline. (Some functions -such as spam detection that were handled in the Mailman 2 pipeline are -now in the filter chains.) +Each of the `command`, `bounce`, and `pipeline` queues is processed by a +*pipeline of handlers* as in Mailman 2's pipeline. (Some functions such as +spam detection that were handled in the Mailman 2 pipeline are now in the +filter chains.) -Handlers may copy messages to other queues (*e.g.*, ARCHIVE), and -eventually posts for distribution end up in the OUT queue for -injection into the MTA. +Handlers may copy messages to other queues (*e.g.*, `archive`), and eventually +posted messages for distribution to the list membership end up in the `out` +queue for injection into the MTA. -The VIRGIN queue is a special queue for messages created by Mailman. +The `virgin` queue is a special queue for messages created by Mailman. .. graphviz:: @@ -69,7 +70,7 @@ The VIRGIN queue is a special queue for messages created by Mailman. "calculate\nrecipients" -> "to digest" -> "to archive" -> \ "to outgoing" } node [shape=box, color=lightblue, style=filled, group=1] - { rank=same; POSTING -> "MIME\ndelete" } + { rank=same; PIPELINE -> "MIME\ndelete" } { rank=same; "to digest" -> DIGEST } { rank=same; "to archive" -> ARCHIVE } { rank=same; "to outgoing" -> OUT } @@ -77,16 +78,15 @@ The VIRGIN queue is a special queue for messages created by Mailman. Message Filtering ------------------ +================= -Once a message has been classified as a post or administrivia, rules -are applied to determine whether the message should be distributed or -acted on. Rules include things like "if the message's sender is a -non-member, hold it for moderation", or "if the message contains an -Approved field with a valid password, distribute it". A rule may also -make no decision, in which case the message pickle is passed on to the -next rule in the filter chain. The default set of rules looks -something like this: +Once a message has been classified as a post or administrivia, rules are +applied to determine whether the message should be distributed or acted on. +Rules include things like "if the message's sender is a non-member, hold it +for moderation", or "if the message contains an `Approved` header with a valid +password, allow it to be posted". A rule may also make no decision, in which +case message processing is passed on to the next rule in the filter chain. +The default set of rules looks something like this: .. graphviz:: @@ -107,12 +107,12 @@ something like this: any [group=0, label="<f0> any | {<f1> | <f2>}"] truth [label="<f0> truth | <f1>"] approved:f1 -> emergency:f0 [weight=100] - emergency:f1 -> loop:f0 - loop:f1 -> modmember:f0 - modmember:f1 -> administrivia:f0 - administrivia:f1 -> maxsize:f0 - maxsize:f1 -> any:f0 - any:f1 -> truth:f0 + emergency:f1 -> loop:f0 + loop:f1 -> modmember:f0 + modmember:f1 -> administrivia:f0 + administrivia:f1 -> maxsize:f0 + maxsize:f1 -> any:f0 + any:f1 -> truth:f0 } subgraph queues { rankdir=TB @@ -122,22 +122,22 @@ something like this: MODERATION [color=wheat]; HOLD [color=wheat]; } - { POSTING [shape=box, style=filled, color=cyan]; } + { PIPELINE [shape=box, style=filled, color=cyan]; } IN -> approved:f0 - approved:f2 -> POSTING [minlen=2] + approved:f2 -> PIPELINE [minlen=2] loop:f2 -> DISCARD modmember:f2 -> MODERATION emergency:f2:e -> HOLD maxsize:f2 -> MODERATION any:f2 -> MODERATION - truth:f1 -> POSTING [minlen=2] + truth:f1 -> PIPELINE [minlen=2] } Configuration -------------- +============= Uses lazr.config. @@ -146,45 +146,44 @@ when the Mailman daemon starts, and what queue the Runner manages. Shell Commands --------------- +============== -bin/mailman: This is an ubercommand, with subcommands for all the -various things admins might want to do, similar to mailmanctl, but -with more functionality. +`bin/mailman`: This is an ubercommand, with subcommands for all the various +things admins might want to do, similar to Mailman 2's mailmanctl, but with +more functionality. -bin/master: The runner manager: starts, watches, stops the runner +`bin/master`: The runner manager: starts, watches, stops the runner daemons. -bin/runner: Individual runner daemons. Each instance is configured -with a configure object specified on the command line, and other -command line options. +`bin/runner`: Individual runner daemons. Each instance is configured with +arguments specified on the command line. User Model ----------- +========== A *user* represents a person. A user has an *id* and a *display -name*, and a list of addresses. +name*, and optionally a list of linked addresses. -Each *address* is a separate object, linked to a user with a user id. +Each *address* is a separate object, linked to no more than one user. -A list *member* is a link from a user to a mailing list. Each list -member has a user id, a mailing list name, an address (which may be -None, which will be replaced by the user's preferred address, a list -of preferences, and a *role* such as "owner" or "moderator". Roles -are used to determine what kinds of mail the user receives via that -membership. *Owners* will receive mail to *list*-owner, but not posts -and moderation traffic, for example. A user with multiple roles on a -single list will therefore have multiple memberships in that list, one -for each role. +A list *member* associates an address with a mailing list. Each list member +has a id, a mailing list name, an address (which may be `None`, representing +the user's *preferred address*), a list of preferences, and a *role* such as +"owner" or "moderator". Roles are used to determine what kinds of mail the +user receives via that membership. *Owners* will receive mail to +*list*-owner, but not posts and moderation traffic, for example. A user with +multiple roles on a single list will therefore have multiple memberships in +that list, one for each role. -Roles are implemented by "magical, invisible" *rosters*. +Roles are implemented by "magical, invisible" *rosters* which are objects +representing queries on the membership database. List Styles ------------ +=========== -Each list *style* is a named object. Its attributes are functions -used to apply the relevant style settings to the mailing list *at -creation time*. Since these are functions, they can be composed in -various ways, to create substyles, *etc*. +Each list *style* is a named object. Its attributes are functions used to +apply the relevant style settings to the mailing list *at creation time*. +Since these are functions, they can be composed in various ways, to create +substyles, *etc*. diff --git a/src/mailman/docs/ArchiveUIin5.rst b/src/mailman/docs/ArchiveUIin5.rst index b66b0b220..58b48cd3d 100644 --- a/src/mailman/docs/ArchiveUIin5.rst +++ b/src/mailman/docs/ArchiveUIin5.rst @@ -1,30 +1,30 @@ -Set Up the Archive UI in Five Minutes +===================================== +Set up the archive ui in five minutes ===================================== -The Hyperkitty application aims at providing an interface to visualize -and explore mailman archives. +The `hyperkitty`_ application aims at providing an interface to visualize and +explore Mailman archives. -This is a `django`_ project. +This is a `Django`_ project. Requirements ------------- +============ -- A mail archive in `maildir format`_ (no, you don't need a running - Mailman 3!) Eventually Hyperkitty will support `mbox format`_ for - backward compatibility with *pipermail*, and *zipped maildirs* seem - like a good idea to save space. **Beware:** Although you'd think - that we would be able to manipulate the venerable *mbox* format - safely and efficiently, that doesn't seem to be the case. *Maildir* - archives are **strongly** preferred, because they are more robust to - program bugs (whether in Mailman, hyperkitty, or in the originating - MUA!) -- `Django`_ is the web framework that supports the UI. +- A mail archive in `maildir format`_ (no, you don't need a running Mailman + 3!) Eventually hyperkitty will support `mbox format`_ for backward + compatibility with *Pipermail*, and *zipped maildirs* seem like a good idea + to save space. **Beware:** Although you'd think that we would be able to + manipulate the venerable *mbox* format safely and efficiently, that doesn't + seem to be the case. *Maildir* archives are **strongly** preferred, because + they are more robust to program bugs (whether in Mailman, hyperkitty, or in + the originating MUA!) +- Django is the web framework that supports the UI. - `bunch`_ DOES WHAT? -- The `notmuch mail indexer`_ is used to generate indices (and - requires `Xapian`_). -- `Hyperkitty`_ itself, which is a UI, and not responsible for - maintaining the message archive itself. (Since the archive is in - `maildir format`_, any modern MTA or MDA can build one for you.) +- The `notmuch mail indexer`_ is used to generate indexes (and requires + `Xapian`_). +- hyperkitty itself, which is a UI, and not responsible for maintaining the + message archive itself. (Since the archive is in `maildir format`_, any + modern MTA or MDA can build one for you.) Get it running (under virtualenv): @@ -35,62 +35,63 @@ environment for your Python applications. - Create the virtualenv:: - virtualenv mailman3 + % virtualenv mailman3 - Activate the virtualenv:: - cd mailman3 - source bin/activate + % cd mailman3 + % source bin/activate -You don't *have* to use virtualenv, though, and if you don't want to, -just omit the preceding steps. Continue with +You don't *have* to use virtualenv, though, and if you don't want to, just +omit the preceding steps. Continue with these steps. -- Install django and dependencies:: +- Install Django and dependencies:: - easy_install django - easy_install bunch + % easy_install django + % easy_install bunch - Install notmuch -- these are bindings that come with the notmuch C library. The easiest way is probably to install them for your OS vendor and then symlink them into the virtualenv similar to this:: - yum install -y python-notmuch + % yum install -y python-notmuch - Note: on a multiarch system like Fedora, the directories may be lib64 rather than lib on 64 bit systems. Next:: - cd lib/python2.7/site-packages - ln -s /usr/lib/python2.7/site-packages/notmuch . + % cd lib/python2.7/site-packages + % ln -s /usr/lib/python2.7/site-packages/notmuch . - Note: this is the version of notmuch I tested with; others may work:: - ln -s /usr/lib/python2.7/site-packages/notmuch-0.11-py2.7.egg-info . + % ln -s /usr/lib/python2.7/site-packages/notmuch-0.11-py2.7.egg-info . - Install the hyperkitty sources:: - git clone http://ambre.pingoured.fr/cgit/hyperkitty.git/ + % git clone http://ambre.pingoured.fr/cgit/hyperkitty.git/ + Running hyperkitty ------------------ - Start it:: - cd hyperkitty + % cd hyperkitty - Put the static content where it should be:: - python manage.py collectstatic + % python manage.py collectstatic - Run the Django server:: - python manage.py runserver + % python manage.py runserver .. _`hyperkitty`: https://launchpad.net/hyperkitty .. _`Django`: http://djangoproject.org/ .. _`notmuch mail indexer`: http://notmuchmail.org .. _`bunch`: http://pypi.python.org/pypi/bunch -.. _`Xapian`: PLEASE-REPORT-MISSING-URL -.. _`maildir format`: PLEASE-REPORT-MISSING-URL -.. _`mbox format`: PLEASE-REPORT-MISSING-URL -.. _`virtualenv`: PLEASE-REPORT-MISSING-URL +.. _`Xapian`: http://xapian.org/ +.. _`maildir format`: http://en.wikipedia.org/wiki/Maildir +.. _`mbox format`: http://en.wikipedia.org/wiki/Mbox +.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 6869e2889..48736b518 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -8,9 +8,37 @@ Copyright (C) 1998-2012 by the Free Software Foundation, Inc. Here is a history of user visible changes to Mailman. +3.0 beta 2 -- "Freeze" +====================== +(20XX-XX-XX) + +Architecture +------------ + * Internally, all datetimes are kept in the UTC timezone, however because of + LP: #280708, they are stored in the database in naive format. + * `received_time` is now added to the message metadata by the LMTP runner + instead of by `Switchboard.enqueue()`. This latter no longer depends on + `received_time` in the metadata. + * The `ArchiveRunner` no longer acquires a lock before it calls the + individual archiver implementations, since not all of them need a lock. If + they do, the implementations must acquire said lock themselves. + +Configuration +------------- + * New configuration variables `clobber_date` and `clobber_skew` supported in + every `[archiver.<name>]` section. These are used to determine under what + circumstances a message destined for a specific archiver should have its + `Date:` header clobbered. (LP: #963612) + +Documentation +------------- + * Some additional documentation on related components such as Postorius and + hyperkitty have been added, given by Stephen J Turnbull. + + 3.0 beta 1 -- "The Twilight Zone" ================================= -(20XX-XX-XX) +(2012-03-23) Architecture ------------ @@ -46,6 +74,18 @@ Architecture attribute on the message object, instead of trusting a possibly incorrect value if it's already set. The individual `IArchiver` implementations no longer set the `X-Message-ID-Hash` header. + * The Prototype archiver now stores its files in maildir format inside of + `$var_dir/archives/prototype`, given by Toshio Kuratomi. + * Improved "8 mile high" document distilled by Stephen J Turnbull from the + Pycon 2012 Mailman 3 sprint. Also improvements to the Sphinx build given + by Andrea Crotti. + * Pipermail has been eradicated. + * Configuration variable `[mailman]filtered_messages_are_preservable` + controls whether messages which have their top-level `Content-Type` + filtered out can be preserved in the `bad` queue by list owners. + * Configuration section `[scrubber]` removed, as is the scrubber handler. + This handler was essentially incompatible with Mailman 3 since it required + coordination with Pipermail to store attachments on disk. Database -------- @@ -60,6 +100,11 @@ Database - digest_footer -> digest_footer_uri - start_chain -> posting_chain - pipeline -> posting_pipeline + - real_name -> display_name (mailinglist, user, address) + * Schema additions: + - mailinglist.filter_action + - mailinglist.owner_chain + - mailinglist.owner_pipeline REST ---- @@ -67,6 +112,8 @@ REST resources now accept a `held` path component. GETing this returns all held messages for the mailing list. POSTing to a specific request id under this url can dispose of the message using `Action` enums. + * Mailing list resources now have a `member_count` attribute which gives the + number of subscribed members. Given by Toshio Kuratomi. Interfaces ---------- @@ -84,6 +131,10 @@ Interfaces * New `ITemplateLoader` utility. * `ILanguageManager.add()` returns the `ILanguage` object just created. * `IMailinglist.decorators` removed; it was unused + * `IMailingList.real_name` -> `IMailingList.display_name` + * `IUser.real_name` -> `IUser.display_name` + * `IAddress.real_name` -> `IAddress.display_name` + * Add property `IRoster.member_count`. Commands -------- @@ -105,6 +156,8 @@ Commands Bug fixes --------- + * Subscription disabled probe warning notification messages are now sent + without a `Precedence:` header. Given by Mark Sapiro. (LP: #808821) * Fixed KeyError in retry runner, contributed by Stephen A. Goss. (LP: #872391) * Fixed bogus use of `bounce_processing` attribute (should have been diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index d0122175c..cd7b5247d 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -8,24 +8,23 @@ Copyright (C) 2008-2012 by the Free Software Foundation, Inc. Beta Release ============ -This is a beta release. The developers believe has sufficient functionality -to provide full services to a mailing list, but it is not ready for -production yet. Interfaces and administration may differ substantially from -the alpha series, but changes should be incremental going forward from beta -1. Changes from the alpha series will be described in notes to the main +This is a beta release. The developers believe it has sufficient +functionality to provide full services to a mailing list, but it is not ready +for production yet. Interfaces and administration may differ substantially +from the alpha series, but changes should be incremental going forward from +beta 1. Changes from the alpha series will be described in notes to the main text. The Mailman 3 beta releases are being provided to give developers and other interested people an early look at the next major version, and site administrators a chance to prepare for an eventual upgrade. The core list management and post distribution functionality is now complete. However, -unlike Mailman 2 whose web interface and archives were tightly integrated -with the core, Mailman 3 exposes a RESTful adminstrative interface to the -web, communicates with archivers via LMTP (or SMTP), and leaves summary, -search, and retrieval of archived messages to a separate application (a -simple implementation is provided). As of beta 1 the web interface and -archiver are still at an early stage of development. As such, some things -may not work. +unlike Mailman 2 whose web interface and archives were tightly integrated with +the core, Mailman 3 exposes a REST administrative interface to the web, +communicates with archivers via decoupled interfaces, and leaves summary, +search, and retrieval of archived messages to a separate application (a simple +implementation is provided). As of beta 1 the web interface and archiver are +still at an early stage of development. As such, some things may not work. Contributions are welcome. Please submit bug reports on the Mailman bug tracker at https://bugs.launchpad.net/mailman though you will currently need @@ -48,10 +47,10 @@ the system ``python`` or an absolute path. Mailman 3 is now based on the `zc.buildout`_ infrastructure, which greatly simplifies building and testing Mailman. -You do not need anything other than Python and an internet connection to get -all the other Mailman 3 dependencies. (N.B. In early betas this is something -of a lie, as the `web UI`_ and `archive UI`_ are distributed and installed -separately.) +During the beta program, you may need some additional dependencies, such as a +C compiler and the Python development headers and libraries. You will need an +internet connection. Also the `web UI`_ and `archive UI`_ are distributed and +installed separately. Building Mailman 3 @@ -75,13 +74,11 @@ Build the online docs by running:: % python setup.py build_sphinx -[In the alphas we used ``bin/docs``, but this does not exist now.] -(You might get warnings which you can safely ignore.) Then visit +(You might get warnings which you can safely ignore.) Then visit:: build/sphinx/html/README.html -in your browser to start reading the documentation. [In the alphas this was -parts/docs/mailman/build/mailman/docs/README.html.] Or you can just read the +in your browser to start reading the documentation. Or you can just read the doctests by looking in all the 'doc' directories under the 'mailman' package. Doctests are documentation first, so they should give you a pretty good idea how various components of Mailman 3 work. @@ -113,6 +110,7 @@ The first existing file found wins. * ``./mailman.cfg`` * ``~/.mailman.cfg`` * ``/etc/mailman.cfg`` + * ``argv[0]/../../etc/mailman.cfg`` Run the ``bin/mailman info`` command to see which configuration file Mailman will use, and where it will put its database file. The first time you run @@ -130,47 +128,43 @@ now, all configuration happens via the command line and REST API. Mailman Web UI -------------- -The Mailman 3 web UI is somewhat coupled to core Mailman via the "RESTful" -client API. It is expected that this architecture will make it possible for -users with special needs to adapt the web UI, or even replace it entirely, -with a reasonable amount of effort. However, as a core feature of Mailman, -the web UI will emphasize usability over modularity at first, so most users -should use the web UI described here. It may not be trivial to use a -different web UI. +The Mailman 3 web UI, called *Postorius*, interfaces to core Mailman engine +via the REST client API. It is expected that this architecture will make it +possible for users with other needs to adapt the web UI, or even replace it +entirely, with a reasonable amount of effort. However, as a core feature of +Mailman, the web UI will emphasize usability over modularity at first, so most +users should use the web UI described here. -The Mailman web UI was prototyped at the PyCon 2012 sprint, so it is "very -very alpha" as of Mailman 3 beta 1, and comes in several components. In -particular, it requires a `Django`_ installation, and Bazaar checkouts of the -`REST client module`_ and the `web ui`_ itself. Building it is fairly -straightforward, however, given Florian Fuchs' `Five Minute Guide` from his -`blog post`_ on the Mailman wiki. (Check the `blog post`_ for the most -recent version!) +Postorius was prototyped at the `Pycon 2012 sprint`_, so it is "very alpha" as +of Mailman 3 beta 1, and comes in several components. In particular, it +requires a `Django`_ installation, and Bazaar checkouts of the `REST client +module`_ and `Postorius`_ itself. Building it is fairly straightforward, +however, given Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the +Mailman wiki. (Check the `blog post`_ for the most recent version!) The List Archiver ----------------- -Experience with Mailman 2's "pipermail" archiver has demonstrated that it is -unnecessary for the archiver to be tightly coupled to core Mailman, and that -it is useful to provide a simple, standard interface for third-party -archiving tools and services. For this reason, Mailman 3 uses the standard -mail transport protocols LMTP and SMTP to forward posts to archivers after -processing for list distribution. Summary, search, and retrieval of archived +In Mailman 3, the archivers are decoupled from the core engine. It is useful +to provide a simple, standard interface for third-party archiving tools and +services. For this reason, Mailman 3 defines a formal interface to insert +messages into any of a number of configured archivers, using whatever protocol +is appropriate for that archiver. Summary, search, and retrieval of archived posts are handled by a separate application. A new `archive UI`_ called Hyperkitty, based on the `notmuch mail indexer`_ -and `Django`_, was prototyped at the PyCon 2012 sprint by Toshio Kuratomi, -and like the web UI it is "very very alpha" as of Mailman 3 beta 1. Unlike -the web UI (and Mailman 2's default "pipermail" archiver), the "hyperkitty" -archiver is very loosely coupled to Mailman 3 core. In fact, any email -application that speaks LMTP or SMTP will be able to use hyperkitty, and -Mailman 3 can use any archiver that accepts posts via LMTP or SMTP. +and `Django`_, was prototyped at the PyCon 2012 sprint by Toshio Kuratomi, and +like the web UI it is also in early alpha as of Mailman 3 beta 1. The +"hyperkitty" archiver is very loosely coupled to Mailman 3 core. In fact, any +email application that speaks LMTP or SMTP will be able to use hyperkitty. A `five minute guide to hyperkitty`_ is based on Toshio Kuratomi's README. + .. _`zc.buildout`: http://pypi.python.org/pypi/zc.buildout .. _`lazr.config`: http://pypi.python.org/pypi/lazr.config -.. _`web ui`: https://launchpad.net/mailmanweb +.. _`Postorius`: https://launchpad.net/postorius .. _`archive UI`: https://launchpad.net/hyperkitty .. _`Django`: http://djangoproject.org/ .. _`REST client module`: https://launchpad.net/mailman.client @@ -178,3 +172,4 @@ A `five minute guide to hyperkitty`_ is based on Toshio Kuratomi's README. .. _`blog post`: http://wiki.list.org/display/DEV/A+5+minute+guide+to+get+the+Mailman+web+UI+running .. _`notmuch mail indexer`: http://notmuchmail.org .. _`five minute guide to hyperkitty`: ArchiveUIin5.html +.. _`Pycon 2012 sprint`: https://us.pycon.org/2012/community/sprints/projects/ diff --git a/src/mailman/docs/WebUIin5.rst b/src/mailman/docs/WebUIin5.rst index 42a2f3e3c..5000de9ad 100644 --- a/src/mailman/docs/WebUIin5.rst +++ b/src/mailman/docs/WebUIin5.rst @@ -1,32 +1,34 @@ -Set Up the Mailman UI in Five Minutes -===================================== +================================ +Set up Postorius in five minutes +================================ -This is a quick guide to set up a development environment to work on -Mailman 3's web UI. If all goes as planned, you should be done within -5 minutes. This has been tested on Ubuntu 11.04. +This is a quick guide for setting up a development environment to work on +Mailman 3's web UI, called Postorius. If all goes as planned, you should be +done within 5 minutes. This has been tested on Ubuntu 11.04. -In order to download the components necessary you need to have the -Bazaar VCS installed on your system. Mailman and mailman.client need -at least Python version 2.6. +In order to download the components necessary you need to have the `Bazaar`_ +version control system installed on your system. Mailman and mailman.client +need at least Python version 2.6. It's probably a good idea to set up a virtual Python environment using `virtualenv`_. `Here is a brief HOWTO`_. -.. _`virtualenv`: PLEASE-REPORT-MISSING-URL +.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv .. _`Here is a brief HOWTO`: ./ArchiveUIin5.html#get-it-running-under-virtualenv +.. _`Bazaar`: http://bazaar.canonical.com/en/ GNU Mailman 3 -------------- +============= -First download the latest revision of Mailman 3 from launchpad. +First download the latest revision of Mailman 3 from Launchpad. :: $ bzr branch lp:mailman Install and test:: - $ cd mailman + $ cd mailman $ python bootstrap.py $ bin/buildout $ bin/test @@ -36,20 +38,20 @@ If you get no errors you can now start Mailman:: $ bin/mailman start $ cd .. -At this point Mailman will not send nor receive any real emails. But -that's fine as long as you only want to work on the components related -to the ReST client or the web ui. +At this point Mailman will not send nor receive any real emails. But that's +fine as long as you only want to work on the components related to the REST +client or the web ui. mailman.client (the Python bindings for Mailman's ReST API) ------------------------------------------------------------ +=========================================================== -Download from launchpad:: +Download from Launchpad:: $ bzr branch lp:mailman.client -Install in development mode to be able to change the code without -working directly on the PYTHONPATH. +Install in development mode to be able to change the code without working +directly on the PYTHONPATH. :: $ cd mailman.client @@ -58,10 +60,10 @@ working directly on the PYTHONPATH. Django >= 1.3 -------------- +============= -The web ui is a pluggable Django application. Therefore you need to -have Django (at least version 1.3) installed. +Postorius is a pluggable Django application. Therefore you need to have +Django (at least version 1.3) installed. :: $ wget http://www.djangoproject.com/download/1.3.1/tarball/ -O Django-1.3.1.tar.gz @@ -71,18 +73,18 @@ have Django (at least version 1.3) installed. $ cd .. -mailmanweb ----------- +Postorius +========= :: - $ bzr branch lp:mailmanweb - $ cd mailmanweb + $ bzr branch lp:postorius + $ cd postorius $ sudo python setup.py develop Start the development server ----------------------------- +============================ :: @@ -92,20 +94,20 @@ Start the development server The last command will start the dev server on http://localhost:8000. + A note for MacOS X users (and possibly others running python 2.7) ------------------------------------------------------------------ +================================================================= *Note: These paragraphs are struck-through on the Mailman wiki.* -On an OS X 10.7 (Lion) system, some of these steps needed to be -modified to use python2.6 instead of python. (In particular, bzr is -known to behave badly when used python2.7 on OS X 10.7 at the moment --- hopefully this will be fixed and no longer an issue soon.) +On an OS X 10.7 (Lion) system, some of these steps needed to be modified to +use python2.6 instead of python. (In particular, bzr is known to behave badly +when used python2.7 on OS X 10.7 at the moment -- hopefully this will be fixed +and no longer an issue soon.) -You will need to install the latest version of XCode on MacOS 10.7, -which is available for free from the app store. If you had a previous -version of XCode installed when you upgraded to 10.7, it will no -longer work and will not have automatically been upgraded, so be -prepared to install again. Once you have it installed from the app -store, you will still need to go run the installer from /Applications -to complete the installation. +You will need to install the latest version of XCode on MacOS 10.7, which is +available for free from the App Store. If you had a previous version of XCode +installed when you upgraded to 10.7, it will no longer work and will not have +automatically been upgraded, so be prepared to install again. Once you have +it installed from the App Store, you will still need to go run the installer +from ``/Applications`` to complete the installation. diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index d7bf81055..dcea82425 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -142,7 +142,7 @@ class Message(email.message.Message): else '') else: field_values = self.get_all(header, []) - senders.extend(address.lower() for (real_name, address) + senders.extend(address.lower() for (display_name, address) in email.utils.getaddresses(field_values)) # Filter out None and the empty string. return [sender for sender in senders if sender] @@ -178,10 +178,20 @@ class UserNotification(Message): self['To'] = recipients self.recipients = set([recipients]) - def send(self, mlist, **_kws): + def send(self, mlist, add_precedence=True, **_kws): """Sends the message by enqueuing it to the 'virgin' queue. This is used for all internally crafted messages. + + :param mlist: The mailing list to send the message to. + :type mlist: `IMailingList` + :param add_precedence: Flag indicating whether a `Precedence: bulk` + header should be added to the message or not. + :type add_precedence: bool + + This function also accepts arbitrary keyword arguments. The key/value + pairs for **kws is added to the metadata dictionary associated with + the enqueued message. """ # Since we're crafting the message from whole cloth, let's make sure # this message has a Message-ID. @@ -193,7 +203,7 @@ class UserNotification(Message): # UserNotifications are typically for admin messages, and for messages # other than list explosions. Send these out as Precedence: bulk, but # don't override an existing Precedence: header. - if 'precedence' not in self: + if 'precedence' not in self and add_precedence: self['Precedence'] = 'bulk' self._enqueue(mlist, **_kws) diff --git a/src/mailman/pipeline/tests/__init__.py b/src/mailman/email/tests/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/mailman/pipeline/tests/__init__.py +++ b/src/mailman/email/tests/__init__.py diff --git a/src/mailman/email/tests/test_message.py b/src/mailman/email/tests/test_message.py new file mode 100644 index 000000000..ee4f6135d --- /dev/null +++ b/src/mailman/email/tests/test_message.py @@ -0,0 +1,60 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the message API.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMessage', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.email.message import UserNotification +from mailman.testing.helpers import get_queue_messages +from mailman.testing.layers import ConfigLayer + + + +class TestMessage(unittest.TestCase): + """Test the message API.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = UserNotification( + 'aperson@example.com', + 'test@example.com', + 'Something you need to know', + 'I needed to tell you this.') + + def test_one_precedence_header(self): + # Ensure that when the original message already has a Precedence: + # header, UserNotification.send(..., add_precedence=True, ...) does + # not add a second header. + self.assertEqual(self._msg['precedence'], None) + self._msg['Precedence'] = 'omg wtf bbq' + self._msg.send(self._mlist) + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msg.get_all('precedence'), + ['omg wtf bbq']) diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index d0df7592f..1861a8121 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""Email address validation.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ diff --git a/src/mailman/handlers/__init__.py b/src/mailman/handlers/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/handlers/__init__.py diff --git a/src/mailman/pipeline/acknowledge.py b/src/mailman/handlers/acknowledge.py index 8a0088ce4..0e0916337 100644 --- a/src/mailman/pipeline/acknowledge.py +++ b/src/mailman/handlers/acknowledge.py @@ -69,20 +69,21 @@ class Acknowledge: else member.preferred_language) charset = language_manager[language.code].charset # Now get the acknowledgement template. - realname = mlist.real_name + display_name = mlist.display_name text = make('postack.txt', mailing_list=mlist, language=language.code, wrap=False, subject=oneline(original_subject, charset), - listname=realname, + list_name=mlist.list_name, + display_name=display_name, listinfo_url=mlist.script_url('listinfo'), optionsurl=member.options_url, ) # Craft the outgoing message, with all headers and attributes # necessary for general delivery. Then enqueue it to the outgoing # queue. - subject = _('$realname post acknowledgment') + subject = _('$display_name post acknowledgment') usermsg = UserNotification(sender, mlist.bounces_address, subject, text, language) usermsg.send(mlist) diff --git a/src/mailman/pipeline/after_delivery.py b/src/mailman/handlers/after_delivery.py index 46007092b..a964804b5 100644 --- a/src/mailman/pipeline/after_delivery.py +++ b/src/mailman/handlers/after_delivery.py @@ -25,12 +25,11 @@ __all__ = [ ] -import datetime - from zope.interface import implements from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now @@ -44,5 +43,5 @@ class AfterDelivery: def process(self, mlist, msg, msgdata): """See `IHander`.""" - mlist.last_post_time = datetime.datetime.now() + mlist.last_post_time = now() mlist.post_id += 1 diff --git a/src/mailman/pipeline/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py index ffbc80c85..ffbc80c85 100644 --- a/src/mailman/pipeline/avoid_duplicates.py +++ b/src/mailman/handlers/avoid_duplicates.py diff --git a/src/mailman/pipeline/cleanse.py b/src/mailman/handlers/cleanse.py index 90f2a892a..605b843d0 100644 --- a/src/mailman/pipeline/cleanse.py +++ b/src/mailman/handlers/cleanse.py @@ -31,8 +31,8 @@ from email.utils import formataddr from zope.interface import implements from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader log = logging.getLogger('mailman.post') diff --git a/src/mailman/pipeline/cleanse_dkim.py b/src/mailman/handlers/cleanse_dkim.py index d2cd32636..d2cd32636 100644 --- a/src/mailman/pipeline/cleanse_dkim.py +++ b/src/mailman/handlers/cleanse_dkim.py diff --git a/src/mailman/pipeline/cook_headers.py b/src/mailman/handlers/cook_headers.py index 2d117429c..2d117429c 100644 --- a/src/mailman/pipeline/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py diff --git a/src/mailman/pipeline/decorate.py b/src/mailman/handlers/decorate.py index 1291a2668..d6d156048 100644 --- a/src/mailman/pipeline/decorate.py +++ b/src/mailman/handlers/decorate.py @@ -57,7 +57,8 @@ def process(mlist, msg, msgdata): d['user_address'] = recipient d['user_delivered_to'] = member.address.original_email d['user_language'] = member.preferred_language.description - d['user_name'] = (member.user.real_name if member.user.real_name + d['user_name'] = (member.user.display_name + if member.user.display_name else member.address.original_email) d['user_optionsurl'] = member.options_url # These strings are descriptive for the log file and shouldn't be i18n'd @@ -215,8 +216,9 @@ def decorate(mlist, uri, extradict=None): # any key/value pairs in the extradict. substitutions = dict( fqdn_listname = mlist.fqdn_listname, - list_name = mlist.real_name, + list_name = mlist.list_name, host_name = mlist.mail_host, + display_name = mlist.display_name, listinfo_uri = mlist.script_url('listinfo'), list_requests = mlist.request_address, description = mlist.description, diff --git a/src/mailman/pipeline/docs/ack-headers.rst b/src/mailman/handlers/docs/ack-headers.rst index dba2169e2..e700e2fd1 100644 --- a/src/mailman/pipeline/docs/ack-headers.rst +++ b/src/mailman/handlers/docs/ack-headers.rst @@ -21,7 +21,7 @@ added. ... A message of great import. ... """) - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, dict(noack=True)) >>> print msg.as_string() From: aperson@example.com diff --git a/src/mailman/pipeline/docs/acknowledge.rst b/src/mailman/handlers/docs/acknowledge.rst index 8c8552190..479aa4ea6 100644 --- a/src/mailman/pipeline/docs/acknowledge.rst +++ b/src/mailman/handlers/docs/acknowledge.rst @@ -8,15 +8,15 @@ acknowledgment. :: >>> mlist = create_list('test@example.com') - >>> mlist.real_name = 'XTest' + >>> mlist.display_name = 'Test' >>> mlist.preferred_language = 'en' >>> # XXX This will almost certainly change once we've worked out the web >>> # space layout for mailing lists now. >>> # Ensure that the virgin queue is empty, since we'll be checking this >>> # for new auto-response messages. - >>> virginq = config.switchboards['virgin'] - >>> virginq.files + >>> from mailman.testing.helpers import get_queue_messages + >>> get_queue_messages('virgin') [] Subscribe a user to the mailing list. @@ -46,7 +46,7 @@ Non-members can't get acknowledgments of their posts to the mailing list. >>> handler = config.handlers['acknowledge'] >>> handler.process(mlist, msg, {}) - >>> virginq.files + >>> get_queue_messages('virgin') [] We can also specify the original sender in the message's metadata. If that @@ -58,7 +58,7 @@ person is also not a member, no acknowledgment will be sent either. ... """) >>> handler.process(mlist, msg, ... dict(original_sender='cperson@example.com')) - >>> virginq.files + >>> get_queue_messages('virgin') [] @@ -72,7 +72,7 @@ Unless the user has requested acknowledgments, they will not get one. ... ... """) >>> handler.process(mlist, msg, {}) - >>> virginq.files + >>> get_queue_messages('virgin') [] Similarly if the original sender is specified in the message metadata, and @@ -87,7 +87,7 @@ will be sent. >>> handler.process(mlist, msg, ... dict(original_sender='dperson@example.com')) - >>> virginq.files + >>> get_queue_messages('virgin') [] @@ -107,23 +107,21 @@ The receipt will include the original message's subject in the response body, ... ... """) >>> handler.process(mlist, msg, {}) - >>> len(virginq.files) + >>> messages = get_queue_messages('virgin') + >>> len(messages) 1 - >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) - >>> virginq.files - [] - >>> dump_msgdata(qdata) + >>> dump_msgdata(messages[0].msgdata) _parsemsg : False listname : test@example.com nodecorate : True recipients : set([u'aperson@example.com']) reduced_list_headers: True ... - >>> print qmsg.as_string() + >>> print messages[0].msg.as_string() ... MIME-Version: 1.0 ... - Subject: XTest post acknowledgment + Subject: Test post acknowledgment From: test-bounces@example.com To: aperson@example.com ... @@ -133,7 +131,7 @@ The receipt will include the original message's subject in the response body, <BLANKLINE> Something witty and insightful <BLANKLINE> - was successfully received by the XTest mailing list. + was successfully received by the Test mailing list. <BLANKLINE> List info page: http://lists.example.com/listinfo/test@example.com Your preferences: http://example.com/aperson@example.com @@ -146,22 +144,20 @@ If there is no subject, then the receipt will use a generic message. ... ... """) >>> handler.process(mlist, msg, {}) - >>> len(virginq.files) + >>> messages = get_queue_messages('virgin') + >>> len(messages) 1 - >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) - >>> virginq.files - [] - >>> dump_msgdata(qdata) + >>> dump_msgdata(messages[0].msgdata) _parsemsg : False listname : test@example.com nodecorate : True recipients : set([u'aperson@example.com']) reduced_list_headers: True ... - >>> print qmsg.as_string() + >>> print messages[0].msg.as_string() MIME-Version: 1.0 ... - Subject: XTest post acknowledgment + Subject: Test post acknowledgment From: test-bounces@example.com To: aperson@example.com ... @@ -171,7 +167,7 @@ If there is no subject, then the receipt will use a generic message. <BLANKLINE> (no subject) <BLANKLINE> - was successfully received by the XTest mailing list. + was successfully received by the Test mailing list. <BLANKLINE> List info page: http://lists.example.com/listinfo/test@example.com Your preferences: http://example.com/aperson@example.com diff --git a/src/mailman/pipeline/docs/after-delivery.rst b/src/mailman/handlers/docs/after-delivery.rst index c3e393cf2..b65b0e77b 100644 --- a/src/mailman/pipeline/docs/after-delivery.rst +++ b/src/mailman/handlers/docs/after-delivery.rst @@ -6,9 +6,10 @@ After a message is delivered, or more correctly, after it has been processed by the rest of the handlers in the incoming queue pipeline, a couple of bookkeeping pieces of information are updated. - >>> import datetime + >>> from datetime import timedelta + >>> from mailman.utilities.datetime import now >>> mlist = create_list('_xtest@example.com') - >>> post_time = datetime.datetime.now() - datetime.timedelta(minutes=10) + >>> post_time = now() - timedelta(minutes=10) >>> mlist.last_post_time = post_time >>> mlist.post_id = 10 diff --git a/src/mailman/pipeline/docs/archives.rst b/src/mailman/handlers/docs/archives.rst index 323d121e8..323d121e8 100644 --- a/src/mailman/pipeline/docs/archives.rst +++ b/src/mailman/handlers/docs/archives.rst diff --git a/src/mailman/pipeline/docs/avoid-duplicates.rst b/src/mailman/handlers/docs/avoid-duplicates.rst index 1e46793c2..1e46793c2 100644 --- a/src/mailman/pipeline/docs/avoid-duplicates.rst +++ b/src/mailman/handlers/docs/avoid-duplicates.rst diff --git a/src/mailman/pipeline/docs/cleanse.rst b/src/mailman/handlers/docs/cleanse.rst index 61dfa8f52..61dfa8f52 100644 --- a/src/mailman/pipeline/docs/cleanse.rst +++ b/src/mailman/handlers/docs/cleanse.rst diff --git a/src/mailman/pipeline/docs/cook-headers.rst b/src/mailman/handlers/docs/cook-headers.rst index e0313f53a..948628d54 100644 --- a/src/mailman/pipeline/docs/cook-headers.rst +++ b/src/mailman/handlers/docs/cook-headers.rst @@ -26,7 +26,7 @@ will place the sender in the message metadata for safe keeping. ... """) >>> msgdata = {} - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, msgdata) >>> print msgdata['original_sender'] aperson@example.com diff --git a/src/mailman/pipeline/docs/decorate.rst b/src/mailman/handlers/docs/decorate.rst index e24e1e252..eae8ea904 100644 --- a/src/mailman/pipeline/docs/decorate.rst +++ b/src/mailman/handlers/docs/decorate.rst @@ -22,7 +22,7 @@ Digest messages get decorated during the digest creation phase so no extra decorations are added for digest messages. :: - >>> from mailman.pipeline.decorate import process + >>> from mailman.handlers.decorate import process >>> process(mlist, msg, dict(isdigest=True)) >>> print msg.as_string() From: aperson@example.org @@ -89,12 +89,12 @@ short descriptive name for the mailing list). :: >>> with open(myheader_path, 'w') as fp: - ... print >> fp, '$list_name header' + ... print >> fp, '$display_name header' >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, '$list_name footer' + ... print >> fp, '$display_name footer' >>> msg = message_from_string(msg_text) - >>> mlist.real_name = 'XTest' + >>> mlist.display_name = 'XTest' >>> process(mlist, msg, {}) >>> print msg.as_string() From: aperson@example.org diff --git a/src/mailman/pipeline/docs/digests.rst b/src/mailman/handlers/docs/digests.rst index d4d563180..d4d563180 100644 --- a/src/mailman/pipeline/docs/digests.rst +++ b/src/mailman/handlers/docs/digests.rst diff --git a/src/mailman/pipeline/docs/file-recips.rst b/src/mailman/handlers/docs/file-recips.rst index 7d157ccc5..7d157ccc5 100644 --- a/src/mailman/pipeline/docs/file-recips.rst +++ b/src/mailman/handlers/docs/file-recips.rst diff --git a/src/mailman/pipeline/docs/filtering.rst b/src/mailman/handlers/docs/filtering.rst index 5b54424e4..fd0b33d3b 100644 --- a/src/mailman/pipeline/docs/filtering.rst +++ b/src/mailman/handlers/docs/filtering.rst @@ -45,7 +45,7 @@ content type matches the filter, the entire message will be discarded. >>> process(mlist, msg, {}) Traceback (most recent call last): ... - DiscardMessage + DiscardMessage: The message's content type was explicitly disallowed However, if we turn off content filtering altogether, then the handler short-circuits. diff --git a/src/mailman/pipeline/docs/calc-recips.rst b/src/mailman/handlers/docs/member-recips.rst index 6dca85816..1439e978f 100644 --- a/src/mailman/pipeline/docs/calc-recips.rst +++ b/src/mailman/handlers/docs/member-recips.rst @@ -6,10 +6,10 @@ Every message that makes it through to the list membership gets sent to a set of recipient addresses. These addresses are calculated by one of the handler modules and depends on a host of factors. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') -Recipients are calculate from the list members, so add a bunch of members to -start out with. First, create a bunch of addresses... +Recipients are calculate from the list membership, so first some people +subscribe to the mailing list... :: >>> from mailman.interfaces.usermanager import IUserManager @@ -35,42 +35,25 @@ start out with. First, create a bunch of addresses... ...then make some of the members digest members. - >>> from mailman.core.constants import DeliveryMode + >>> from mailman.interfaces.member import DeliveryMode >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests -Short-circuiting -================ +Regular delivery recipients +=========================== -Sometimes, the list of recipients already exists in the message metadata. -This can happen for example, when a message was previously delivered to some -but not all of the recipients. -:: +Regular delivery recipients are those people who get messages from the list as +soon as they are posted. In other words, these folks are not digest members. >>> msg = message_from_string("""\ ... From: Xavier Person <xperson@example.com> ... ... Something of great import. ... """) - >>> recipients = set(('qperson@example.com', 'zperson@example.com')) - >>> msgdata = dict(recipients=recipients) - - >>> handler = config.handlers['calculate-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - qperson@example.com - zperson@example.com - - -Regular delivery recipients -=========================== - -Regular delivery recipients are those people who get messages from the list as -soon as they are posted. In other words, these folks are not digest members. - >>> msgdata = {} + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> dump_list(msgdata['recipients']) aperson@example.com diff --git a/src/mailman/pipeline/docs/nntp.rst b/src/mailman/handlers/docs/nntp.rst index 874712397..874712397 100644 --- a/src/mailman/pipeline/docs/nntp.rst +++ b/src/mailman/handlers/docs/nntp.rst diff --git a/src/mailman/handlers/docs/owner-recips.rst b/src/mailman/handlers/docs/owner-recips.rst new file mode 100644 index 000000000..e62551ba6 --- /dev/null +++ b/src/mailman/handlers/docs/owner-recips.rst @@ -0,0 +1,63 @@ +===================== +List owner recipients +===================== + +When a message is posted to a mailing list's `-owners` address, all of the +list's administrators will receive a copy. The administrators are defined as +the set of owners and moderators. + + >>> mlist_1 = create_list('alpha@example.com') + +Anne is the owner of the list and Bart is a moderator of the list. + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + >>> anne_addr = user_manager.create_address('anne@example.com') + >>> bart_addr = user_manager.create_address('bart@example.com') + >>> from mailman.interfaces.member import MemberRole + >>> anne = mlist_1.subscribe(anne_addr, MemberRole.owner) + >>> bart = mlist_1.subscribe(bart_addr, MemberRole.moderator) + +The recipients list for the `-owners` address includes both Anne and Bart. + + >>> msg = message_from_string("""\ + ... From: Xavier Person <xperson@example.com> + ... To: alpha@example.com + ... + ... """) + >>> msgdata = {} + >>> handler = config.handlers['owner-recipients'] + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + anne@example.com + bart@example.com + +Anne disables her owner delivery, so she will not receive `-owner` emails. + + >>> from mailman.interfaces.member import DeliveryStatus + >>> anne.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + bart@example.com + +If Bart also disables his owner delivery, then no one could contact the list's +owners. Since this is unacceptable, the site owner is used as a fallback. + + >>> bart.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com + +For mailing lists which have no owners at all, the site owner is also used as +a fallback. + + >>> mlist_2 = create_list('beta@example.com') + >>> mlist_2.administrators.member_count + 0 + >>> msgdata = {} + >>> handler.process(mlist_2, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com diff --git a/src/mailman/pipeline/docs/reply-to.rst b/src/mailman/handlers/docs/reply-to.rst index e08fea81d..d421e2dc5 100644 --- a/src/mailman/pipeline/docs/reply-to.rst +++ b/src/mailman/handlers/docs/reply-to.rst @@ -46,7 +46,7 @@ original message, the list's posting address simply gets inserted. ... ... """) - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, {}) >>> len(msg.get_all('reply-to')) 1 diff --git a/src/mailman/pipeline/docs/replybot.rst b/src/mailman/handlers/docs/replybot.rst index 208f6aae9..2793e4f75 100644 --- a/src/mailman/pipeline/docs/replybot.rst +++ b/src/mailman/handlers/docs/replybot.rst @@ -8,7 +8,7 @@ responses are subject to various conditions, such as headers in the original message or the amount of time since the last auto-response. >>> mlist = create_list('_xtest@example.com') - >>> mlist.real_name = 'XTest' + >>> mlist.display_name = 'XTest' Basic automatic responding @@ -21,11 +21,11 @@ automatic response grace period which specifies how much time must pass before a second response will be sent, with 0 meaning "there is no grace period". :: - >>> import datetime + >>> from datetime import timedelta >>> from mailman.interfaces.autorespond import ResponseAction >>> mlist.autorespond_owner = ResponseAction.respond_and_continue - >>> mlist.autoresponse_grace_period = datetime.timedelta() + >>> mlist.autoresponse_grace_period = timedelta() >>> mlist.autoresponse_owner_text = 'owner autoresponse text' >>> msg = message_from_string("""\ @@ -242,7 +242,7 @@ Automatic responses have a grace period, during which no additional responses will be sent. This is so as not to bombard the sender with responses. The grace period is measured in days. - >>> mlist.autoresponse_grace_period = datetime.timedelta(days=10) + >>> mlist.autoresponse_grace_period = timedelta(days=10) When a response is sent to a person via any of the owner, request, or postings addresses, the response date is recorded. The grace period is usually diff --git a/src/mailman/pipeline/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst index a1ba6c746..0461f27ba 100644 --- a/src/mailman/pipeline/docs/rfc-2369.rst +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -28,7 +28,7 @@ headers generally start with the `List-` prefix. The `rfc-2369` handler adds the `List-` headers. `List-Id` is always added. - >>> from mailman.pipeline.rfc_2369 import process + >>> from mailman.handlers.rfc_2369 import process >>> msg = message_from_string("""\ ... From: aperson@example.com ... @@ -148,35 +148,11 @@ header will be added. >>> mlist.archive = True - >>> from mailman.config import config - >>> config.push('pipermail', """ - ... [archiver.prototype] - ... enable: no - ... [archiver.mail_archive] - ... enable: no - ... [archiver.mhonarc] - ... enable: no - ... [archiver.pipermail] - ... enable: yes - ... """) - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg, only='list-archive') - ---start--- - list-archive: <http://www.example.com/pipermail/test@example.com> - ---end--- - `RFC 5064`_ defines the `Archived-At` header which contains the url to the individual message in the archives. Archivers which don't support -pre-calculation of the archive url cannot add the `Archived-At` header, as is -the case with Pipermail (see above). However, other archivers can calculate -the url, and do add this header. +pre-calculation of the archive url cannot add the `Archived-At` header. +However, other archivers can calculate the url, and do add this header. - >>> config.pop('pipermail') >>> config.push('prototype', """ ... [archiver.prototype] ... enable: yes diff --git a/src/mailman/pipeline/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst index e7a6553ce..48cee8e2b 100644 --- a/src/mailman/pipeline/docs/subject-munging.rst +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -29,7 +29,7 @@ subject munging, a mailing list must have a preferred language. ... """) >>> msgdata = {} - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, msgdata) The original subject header is stored in the message metadata. We must print diff --git a/src/mailman/pipeline/docs/tagger.rst b/src/mailman/handlers/docs/tagger.rst index 80e682119..b64b05c54 100644 --- a/src/mailman/pipeline/docs/tagger.rst +++ b/src/mailman/handlers/docs/tagger.rst @@ -26,7 +26,7 @@ are defined. ... """) >>> msgdata = {} - >>> from mailman.pipeline.tagger import process + >>> from mailman.handlers.tagger import process >>> process(mlist, msg, msgdata) >>> print msg.as_string() Subject: foobar diff --git a/src/mailman/pipeline/docs/to-outgoing.rst b/src/mailman/handlers/docs/to-outgoing.rst index 816aa4ca6..816aa4ca6 100644 --- a/src/mailman/pipeline/docs/to-outgoing.rst +++ b/src/mailman/handlers/docs/to-outgoing.rst diff --git a/src/mailman/pipeline/file_recipients.py b/src/mailman/handlers/file_recipients.py index d087ff2bb..d087ff2bb 100644 --- a/src/mailman/pipeline/file_recipients.py +++ b/src/mailman/handlers/file_recipients.py diff --git a/src/mailman/pipeline/calculate_recipients.py b/src/mailman/handlers/member_recipients.py index 6118a4561..956ea6adc 100644 --- a/src/mailman/pipeline/calculate_recipients.py +++ b/src/mailman/handlers/member_recipients.py @@ -23,13 +23,14 @@ on the `recipients' attribute of the message. This attribute is used by the SendmailDeliver and BulkDeliver modules. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ - 'CalculateRecipients', + 'MemberRecipients', ] + from zope.interface import implements from mailman.config import config @@ -41,12 +42,12 @@ from mailman.utilities.string import wrap -class CalculateRecipients: +class MemberRecipients: """Calculate the regular (i.e. non-digest) recipients of the message.""" implements(IHandler) - name = 'calculate-recipients' + name = 'member-recipients' description = _('Calculate the regular recipients of the message.') def process(self, mlist, msg, msgdata): @@ -82,10 +83,9 @@ class CalculateRecipients: # Bad Urgent: password, so reject it instead of passing it on. # I think it's better that the sender know they screwed up # than to deliver it normally. - realname = mlist.real_name text = _("""\ -Your urgent message to the $realname mailing list was not authorized for -delivery. The original message as received by Mailman is attached. +Your urgent message to the $mlist.display_name mailing list was not authorized +for delivery. The original message as received by Mailman is attached. """) raise errors.RejectMessage(wrap(text)) # Calculate the regular recipients of the message diff --git a/src/mailman/pipeline/mime_delete.py b/src/mailman/handlers/mime_delete.py index 402d13714..c9c1eb408 100644 --- a/src/mailman/pipeline/mime_delete.py +++ b/src/mailman/handlers/mime_delete.py @@ -37,14 +37,18 @@ import errno import logging import tempfile -from email.Iterators import typed_subpart_iterator +from email.iterators import typed_subpart_iterator +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from lazr.config import as_boolean from os.path import splitext from zope.interface import implements from mailman.config import config from mailman.core import errors from mailman.core.i18n import _ -from mailman.core.switchboard import Switchboard +from mailman.email.message import OwnerNotification +from mailman.interfaces.action import FilterAction from mailman.interfaces.handler import IHandler from mailman.utilities.string import oneline from mailman.version import VERSION @@ -54,6 +58,43 @@ log = logging.getLogger('mailman.error') +def dispose(mlist, msg, msgdata, why): + if mlist.filter_action is FilterAction.reject: + # Bounce the message to the original author. + raise errors.RejectMessage(why) + elif mlist.filter_action is FilterAction.forward: + # Forward it on to the list moderators. + text=_("""\ +The attached message matched the $mlist.display_name mailing list's content +filtering rules and was prevented from being forwarded on to the list +membership. You are receiving the only remaining copy of the discarded +message. + +""") + subject=_('Content filter message notification') + notice = OwnerNotification(mlist, subject, roster=mlist.moderators) + notice.set_type('multipart/mixed') + notice.attach(MIMEText(text)) + notice.attach(MIMEMessage(msg)) + notice.send(mlist) + # Let this fall through so the original message gets discarded. + elif mlist.filter_action is FilterAction.preserve: + if as_boolean(config.mailman.filtered_messages_are_preservable): + # This is just like discarding the message except that a copy is + # placed in the 'bad' queue should the site administrator want to + # inspect the message. + filebase = config.switchboards['bad'].enqueue(msg, msgdata) + log.info('{0} preserved in file base {1}'.format( + msg.get('message-id', 'n/a'), filebase)) + else: + log.error( + '{1} invalid FilterAction: {0}. Treating as discard'.format( + mlist.fqdn_listname, mlist.filter_action.name)) + # Most cases also discard the message + raise errors.DiscardMessage(why) + + + def process(mlist, msg, msgdata): # We also don't care about our own digests or plaintext ctype = msg.get_content_type() @@ -227,31 +268,6 @@ def to_plaintext(msg): -def dispose(mlist, msg, msgdata, why): - # filter_action == 0 just discards, see below - if mlist.filter_action == 1: - # Bounce the message to the original author - raise errors.RejectMessage, why - if mlist.filter_action == 2: - # Forward it on to the list owner - listname = mlist.internal_name() - mlist.ForwardMessage( - msg, - text=_("""\ -The attached message matched the $listname mailing list's content filtering -rules and was prevented from being forwarded on to the list membership. You -are receiving the only remaining copy of the discarded message. - -"""), - subject=_('Content filtered message notification')) - if mlist.filter_action == 3 and \ - config.OWNERS_CAN_PRESERVE_FILTERED_MESSAGES: - badq = Switchboard('bad', config.BADQUEUE_DIR) - badq.enqueue(msg, msgdata) - # Most cases also discard the message - raise errors.DiscardMessage - - def get_file_ext(m): """ Get filename extension. Caution: some virus don't put filename diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py new file mode 100644 index 000000000..e431d00cf --- /dev/null +++ b/src/mailman/handlers/owner_recipients.py @@ -0,0 +1,67 @@ +# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Calculate the list owner recipients (includes moderators).""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'OwnerRecipients', + ] + + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import DeliveryStatus + + + +class OwnerRecipients: + """Calculate the owner (and moderator) recipients for -owner postings.""" + + implements(IHandler) + + name = 'owner-recipients' + description = _('Calculate the owner and moderator recipients.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit if we've already calculated the recipients list, + # regardless of whether the list is empty or not. + if 'recipients' in msgdata: + return + # -owner messages go to both the owners and moderators, which is most + # conveniently accessed via the administrators roster. + recipients = set(admin.address.email + for admin in mlist.administrators.members + if admin.delivery_status == DeliveryStatus.enabled) + # To prevent -owner messages from going into a black hole, if there + # are no administrators available, the message goes to the site owner. + if len(recipients) == 0: + msgdata['recipients'] = set((config.mailman.site_owner,)) + else: + msgdata['recipients'] = recipients + # Don't decorate these messages with the header/footers. Eventually + # we should support unique decorations for owner emails. + msgdata['nodecorate'] = True + # We should probably always VERP deliveries to the owners. We + # *really* want to know if they are bouncing. + msgdata['verp'] = True diff --git a/src/mailman/pipeline/replybot.py b/src/mailman/handlers/replybot.py index fc58792e8..83aa40214 100644 --- a/src/mailman/pipeline/replybot.py +++ b/src/mailman/handlers/replybot.py @@ -101,14 +101,16 @@ class Replybot: return # Okay, we know we're going to respond to this sender, craft the # message, send it, and update the database. - realname = mlist.real_name + display_name = mlist.display_name subject = _( - 'Auto-response for your message to the "$realname" mailing list') + 'Auto-response for your message to the "$display_name" ' + 'mailing list') # Do string interpolation into the autoresponse text - d = dict(listname = realname, - listurl = mlist.script_url('listinfo'), - requestemail = mlist.request_address, - owneremail = mlist.owner_address, + d = dict(list_name = mlist.list_name, + display_name = display_name, + listurl = mlist.script_url('listinfo'), + requestemail = mlist.request_address, + owneremail = mlist.owner_address, ) # Interpolation and Wrap the response text. text = wrap(expand(response_text, d)) diff --git a/src/mailman/pipeline/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index 26bfe094c..ece4e83cb 100644 --- a/src/mailman/pipeline/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -30,8 +30,9 @@ from zope.interface import implements from mailman.config import config from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader + CONTINUATION = ',\n\t' diff --git a/src/mailman/pipeline/tagger.py b/src/mailman/handlers/tagger.py index 49e004a12..49e004a12 100644 --- a/src/mailman/pipeline/tagger.py +++ b/src/mailman/handlers/tagger.py diff --git a/src/mailman/handlers/tests/__init__.py b/src/mailman/handlers/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/handlers/tests/__init__.py diff --git a/src/mailman/handlers/tests/test_mimedel.py b/src/mailman/handlers/tests/test_mimedel.py new file mode 100644 index 000000000..6ca34b17b --- /dev/null +++ b/src/mailman/handlers/tests/test_mimedel.py @@ -0,0 +1,213 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the mime_delete handler.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestDispose', + ] + + +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.core import errors +from mailman.handlers import mime_delete +from mailman.interfaces.action import FilterAction +from mailman.interfaces.member import MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import ( + LogFileMark, + get_queue_messages, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestDispose(unittest.TestCase): + """Test the mime_delete handler.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Subject: A disposable message +Message-ID: <ant> + +""") + # Python 2.7 has assertMultiLineEqual. Let this work without bounds. + self.maxDiff = None + self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + config.push('dispose', """ + [mailman] + site_owner: noreply@example.com + """) + + def tearDown(self): + config.pop('dispose') + + def test_dispose_discard(self): + self._mlist.filter_action = FilterAction.discard + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'discarding') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'discarding') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_bounce(self): + self._mlist.filter_action = FilterAction.reject + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'rejecting') + except errors.RejectMessage as error: + pass + else: + raise AssertionError('RejectMessage exception expected') + self.assertEqual(error.message, 'rejecting') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_forward(self): + # The disposed message gets forwarded to the list moderators. So + # first add some moderators. + user_manager = getUtility(IUserManager) + anne = user_manager.create_address('anne@example.com') + bart = user_manager.create_address('bart@example.com') + self._mlist.subscribe(anne, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.moderator) + # Now set the filter action and dispose the message. + self._mlist.filter_action = FilterAction.forward + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'forwarding') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'forwarding') + # There should now be a multipart message in the virgin queue destined + # for the mailing list owners. + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message.get_content_type(), 'multipart/mixed') + # Anne and Bart should be recipients of the message, but it will look + # like the message is going to the list owners. + self.assertEqual(message['to'], 'test-owner@example.com') + self.assertEqual(message.recipients, + set(['anne@example.com', 'bart@example.com'])) + # The list owner should be the sender. + self.assertEqual(message['from'], 'noreply@example.com') + self.assertEqual(message['subject'], + 'Content filter message notification') + # The body of the first part provides the moderators some details. + part0 = message.get_payload(0) + self.assertEqual(part0.get_content_type(), 'text/plain') + self.eq(part0.get_payload(), """\ +The attached message matched the Test mailing list's content +filtering rules and was prevented from being forwarded on to the list +membership. You are receiving the only remaining copy of the discarded +message. + +""") + # The second part is the container for the original message. + part1 = message.get_payload(1) + self.assertEqual(part1.get_content_type(), 'message/rfc822') + # And the first part of *that* message will be the original message. + original = part1.get_payload(0) + self.assertEqual(original['subject'], 'A disposable message') + self.assertEqual(original['message-id'], '<ant>') + + def test_dispose_non_preservable(self): + # Two actions can happen here, depending on a site-wide setting. If + # the site owner has indicated that filtered messages cannot be + # preserved, then this is the same as discarding them. + self._mlist.filter_action = FilterAction.preserve + config.push('non-preservable', """ + [mailman] + filtered_messages_are_preservable: no + """) + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'not preserved') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + finally: + config.pop('non-preservable') + self.assertEqual(error.message, 'not preserved') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_preservable(self): + # Two actions can happen here, depending on a site-wide setting. If + # the site owner has indicated that filtered messages can be + # preserved, then this is similar to discarding the message except + # that a copy is preserved in the 'bad' queue. + self._mlist.filter_action = FilterAction.preserve + config.push('preservable', """ + [mailman] + filtered_messages_are_preservable: yes + """) + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'preserved') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + finally: + config.pop('preservable') + self.assertEqual(error.message, 'preserved') + # There should be no messages in the 'bad' queue. + messages = get_queue_messages('bad') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message['subject'], 'A disposable message') + self.assertEqual(message['message-id'], '<ant>') + + def test_bad_action(self): + # This should never happen, but what if it does? + # FilterAction.accept, FilterAction.hold, and FilterAction.defer are + # not valid. They are treated as discard actions, but the problem is + # also logged. + for action in (FilterAction.accept, + FilterAction.hold, + FilterAction.defer): + self._mlist.filter_action = action + mark = LogFileMark('mailman.error') + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'bad action') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'bad action') + line = mark.readline()[:-1] + self.assertTrue(line.endswith( + '{0} invalid FilterAction: test@example.com. ' + 'Treating as discard'.format(action.name))) diff --git a/src/mailman/handlers/tests/test_recipients.py b/src/mailman/handlers/tests/test_recipients.py new file mode 100644 index 000000000..29cd5b64a --- /dev/null +++ b/src/mailman/handlers/tests/test_recipients.py @@ -0,0 +1,200 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Testing various recipients stuff.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMemberRecipients', + 'TestOwnerRecipients', + ] + + +import unittest + +from zope.component import getUtility +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + + +class TestMemberRecipients(unittest.TestCase): + """Test regular member recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.member) + self._dave = self._mlist.subscribe(dave, MemberRole.member) + self._process = config.handlers['member-recipients'].process + self._msg = mfs("""\ +From: Elle Person <elle@example.com> +To: test@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the list membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's regular members. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'cris@example.com', + 'dave@example.com'))) + + def test_digest_members_not_included(self): + # Digest members are not included in the recipients calculated by this + # handler. + self._cris.preferences.delivery_mode = DeliveryMode.mime_digests + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'dave@example.com'))) + + + +class TestOwnerRecipients(unittest.TestCase): + """Test owner recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + # Make Cris and Dave owners of the mailing list. + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.owner) + self._dave = self._mlist.subscribe(dave, MemberRole.owner) + self._process = config.handlers['owner-recipients'].process + self._msg = mfs("""\ +From: Elle Person <elle@example.com> +To: test-owner@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the owner membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's owners. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + + def test_with_moderators(self): + # Moderators are included in the owner recipient list. + elle = self._manager.create_address('elle@example.com') + fred = self._manager.create_address('fred@example.com') + gwen = self._manager.create_address('gwen@example.com') + self._mlist.subscribe(elle, MemberRole.moderator) + self._mlist.subscribe(fred, MemberRole.moderator) + self._mlist.subscribe(gwen, MemberRole.owner) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com', + 'elle@example.com', + 'fred@example.com', + 'gwen@example.com'))) + + def test_dont_decorate(self): + # Messages to the administrators don't get decorated. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertTrue(msgdata['nodecorate']) + + def test_omit_disabled_owners(self): + # Owner memberships can be disabled, and these folks will not get the + # messages. + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_include_membership_disabled_owner_enabled(self): + # If an address is subscribed to a mailing list as both an owner and a + # member, and their membership is disabled but their ownership + # subscription is not, they still get owner email. + dave = self._manager.get_address('dave@example.com') + member = self._mlist.subscribe(dave, MemberRole.member) + member.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + # Dave disables his owner membership but re-enables his list + # membership. He will not get the owner emails now. + member.preferences.delivery_status = DeliveryStatus.enabled + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_all_owners_disabled(self): + # If all the owners are disabled, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.preferences.delivery_status = DeliveryStatus.by_user + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) + + def test_no_owners(self): + # If a list has no owners or moderators, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.unsubscribe() + self._dave.unsubscribe() + self.assertEqual(self._mlist.administrators.member_count, 0) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) diff --git a/src/mailman/pipeline/to_archive.py b/src/mailman/handlers/to_archive.py index fd5259a14..fd5259a14 100644 --- a/src/mailman/pipeline/to_archive.py +++ b/src/mailman/handlers/to_archive.py diff --git a/src/mailman/pipeline/to_digest.py b/src/mailman/handlers/to_digest.py index 698f16e1e..71511f136 100644 --- a/src/mailman/pipeline/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -26,7 +26,6 @@ __all__ = [ import os -import datetime from zope.interface import implements @@ -35,6 +34,7 @@ from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now as right_now from mailman.utilities.mailbox import Mailbox @@ -85,7 +85,7 @@ class ToDigest: def bump_digest_number_and_volume(mlist): """Bump the digest number and volume.""" - now = datetime.datetime.now() + now = right_now() if mlist.digest_last_sent_at is None: # There has been no previous digest. bump = False diff --git a/src/mailman/pipeline/to_outgoing.py b/src/mailman/handlers/to_outgoing.py index 971f87757..971f87757 100644 --- a/src/mailman/pipeline/to_outgoing.py +++ b/src/mailman/handlers/to_outgoing.py diff --git a/src/mailman/pipeline/to_usenet.py b/src/mailman/handlers/to_usenet.py index 26a383c64..26a383c64 100644 --- a/src/mailman/pipeline/to_usenet.py +++ b/src/mailman/handlers/to_usenet.py diff --git a/src/mailman/interfaces/action.py b/src/mailman/interfaces/action.py index c7c79f7d8..9b3c1fbcc 100644 --- a/src/mailman/interfaces/action.py +++ b/src/mailman/interfaces/action.py @@ -20,6 +20,7 @@ __metaclass__ = type __all__ = [ 'Action', + 'FilterAction', ] @@ -33,3 +34,8 @@ class Action(Enum): discard = 2 accept = 3 defer = 4 + + +class FilterAction(Action): + forward = 5 + preserve = 6 diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py index c26fb92c9..cf2c50bf4 100644 --- a/src/mailman/interfaces/address.py +++ b/src/mailman/interfaces/address.py @@ -17,13 +17,14 @@ """Interface for email address related information.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'AddressAlreadyLinkedError', 'AddressError', 'AddressNotLinkedError', + 'EmailError', 'ExistingAddressError', 'IAddress', 'IEmailValidator', @@ -37,15 +38,30 @@ from mailman.interfaces.errors import MailmanError +class EmailError(MailmanError): + """A generic text email address-related error occurred.""" + + def __init__(self, email): + super(EmailError, self).__init__() + self.email = email + + def __str__(self): + # This is a workaround for Python 2.6 support. When self.email + # contains non-ascii characters, this will cause unprintable output in + # doctests. Python 2.7 can handle it but we haven't dropped support + # for 2.6 yet. + return self.email.encode('us-ascii', 'backslashreplace') + + class AddressError(MailmanError): - """A general address-related error occurred.""" + """A generic IAddress-related error occurred.""" def __init__(self, address): super(AddressError, self).__init__() self.address = address def __str__(self): - return self.address + return str(self.address) class ExistingAddressError(AddressError): @@ -60,7 +76,7 @@ class AddressNotLinkedError(AddressError): """The address is not linked to the user.""" -class InvalidEmailAddressError(AddressError): +class InvalidEmailAddressError(EmailError): """Email address is invalid.""" @@ -84,8 +100,8 @@ class IAddress(Interface): case preserved email address; `email` will always be lower case. """) - real_name = Attribute( - """Optional real name associated with the email address.""") + display_name = Attribute( + """Optional display name associated with the email address.""") registered_on = Attribute( """The date and time at which this email address was registered. diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index f24e44183..f3edc7719 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -21,13 +21,20 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'ClobberDate', 'IArchiver', - 'IPipermailMailingList', ] +from flufl.enum import Enum from zope.interface import Interface, Attribute -from mailman.interfaces.mailinglist import IMailingList + + + +class ClobberDate(Enum): + never = 1 + maybe = 2 + always = 3 @@ -43,36 +50,25 @@ class IArchiver(Interface): :returns: The url string. """ - def permalink(mlist, message): + def permalink(mlist, msg): """Return the url to the message in the archive. This url points directly to the message in the archive. This method only calculates the url, it does not actually archive the message. :param mlist: The IMailingList object. - :param message: The message object. + :param msg: The message object. :returns: The url string or None if the message's archive url cannot be calculated. """ - def archive_message(mlist, message): + def archive_message(mlist, msg): """Send the message to the archiver. :param mlist: The IMailingList object. - :param message: The message object. + :param msg: The message object. :returns: The url string or None if the message's archive url cannot be calculated. """ # XXX How to handle attachments? - - - -class IPipermailMailingList(IMailingList): - """An interface that adapts IMailingList as needed for Pipermail.""" - - def archive_dir(): - """The directory for storing Pipermail artifacts. - - Pipermail expects this to be a function, not a property. - """ diff --git a/src/mailman/interfaces/handler.py b/src/mailman/interfaces/handler.py index f9a075b8c..9007e8490 100644 --- a/src/mailman/interfaces/handler.py +++ b/src/mailman/interfaces/handler.py @@ -17,7 +17,7 @@ """Interface describing a pipeline handler.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index a3e6e443a..bced070d3 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -91,11 +91,9 @@ class IMailingList(Interface): domain = Attribute( """The `IDomain` that this mailing list is defined in.""") - real_name = Attribute("""\ - The short human-readable descriptive name for the mailing list. By - default, this is the capitalized `list_name`, but it can be changed to - anything. This is used in locations such as the message footers and - Subject prefix. + display_name = Attribute("""\ + The short human-readable descriptive name for the mailing list. This + is used in locations such as the message footers and Subject prefix. """) description = Attribute("""\ @@ -384,7 +382,7 @@ class IMailingList(Interface): # Processing. posting_chain = Attribute( - """This mailing list's moderation chain. + """This mailing list's posting moderation chain. When messages are posted to a mailing list, it first goes through a moderation chain to determine whether the message will be accepted. @@ -399,6 +397,24 @@ class IMailingList(Interface): This attribute names a pipeline for postings, which must exist. """) + owner_chain = Attribute( + """This mailing list's owner moderation chain. + + When messages are posted to the owners of a mailing list, it first + goes through a moderation chain to determine whether the message will + be accepted. This attribute names a chain for postings, which must + exist. + """) + + owner_pipeline = Attribute( + """This mailing list's owner posting pipeline. + + Every mailing list has a processing pipeline that messages flow + through once they've been accepted for posting to the owners of a + mailing list. This attribute names a pipeline for postings, which + must exist. + """) + data_path = Attribute( """The file system path to list-specific data. @@ -423,6 +439,12 @@ class IMailingList(Interface): Filtering is performed on MIME type and file name extension. """) + filter_action = Attribute( + """Action to take when the top-level content-type is filtered. + + The value is a `FilterAction` enum. + """) + convert_html_to_plaintext = Attribute( """Flag specifying whether text/html parts should be converted. diff --git a/src/mailman/interfaces/registrar.py b/src/mailman/interfaces/registrar.py index 0f4fe2edf..f00e167ff 100644 --- a/src/mailman/interfaces/registrar.py +++ b/src/mailman/interfaces/registrar.py @@ -42,7 +42,7 @@ class IRegistrar(Interface): syntax checking, or confirmation, while this interface does. """ - def register(mlist, email, real_name=None, delivery_mode=None): + def register(mlist, email, display_name=None, delivery_mode=None): """Register the email address, requesting verification. No `IAddress` or `IUser` is created during this step, but after @@ -58,8 +58,8 @@ class IRegistrar(Interface): :type mlist: `IMailingList` :param email: The email address to register. :type email: str - :param real_name: The optional real name of the user. - :type real_name: str + :param display_name: The optional display name of the user. + :type display_name: str :param delivery_mode: The optional delivery mode for this registration. If not given, regular delivery is used. :type delivery_mode: `DeliveryMode` diff --git a/src/mailman/interfaces/roster.py b/src/mailman/interfaces/roster.py index 4ec3c611c..ebe057d21 100644 --- a/src/mailman/interfaces/roster.py +++ b/src/mailman/interfaces/roster.py @@ -40,6 +40,9 @@ class IRoster(Interface): members = Attribute( """An iterator over all the IMembers managed by this roster.""") + member_count = Attribute( + """The number of members managed by this roster.""") + users = Attribute( """An iterator over all the IUsers reachable by this roster. diff --git a/src/mailman/interfaces/subscriptions.py b/src/mailman/interfaces/subscriptions.py index 584ca4132..85f333cf8 100644 --- a/src/mailman/interfaces/subscriptions.py +++ b/src/mailman/interfaces/subscriptions.py @@ -92,7 +92,7 @@ class ISubscriptionService(Interface): def __iter__(): """See `get_members()`.""" - def join(fqdn_listname, subscriber, real_name=None, + def join(fqdn_listname, subscriber, display_name=None, delivery_mode=DeliveryMode.regular, role=MemberRole.member): """Subscribe to a mailing list. @@ -109,10 +109,10 @@ class ISubscriptionService(Interface): :param subscriber: The email address or user id of the user getting subscribed. :type subscriber: string or int - :param real_name: The name of the user. This is only used if a new + :param display_name: The name of the user. This is only used if a new user is created, and it defaults to the local part of the email address if not given. - :type real_name: string + :type display_name: string :param delivery_mode: The delivery mode for this subscription. This can be one of the enum values of `DeliveryMode`. If not given, regular delivery is assumed. diff --git a/src/mailman/interfaces/user.py b/src/mailman/interfaces/user.py index ad1ac9282..8d0cbfb54 100644 --- a/src/mailman/interfaces/user.py +++ b/src/mailman/interfaces/user.py @@ -17,7 +17,7 @@ """Interface describing the basics of a user.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,27 +28,20 @@ __all__ = [ from zope.interface import Interface, Attribute -from mailman.interfaces.errors import MailmanError +from mailman.interfaces.address import AddressError -class UnverifiedAddressError(MailmanError): +class UnverifiedAddressError(AddressError): """Unverified address cannot be used as a user's preferred address.""" - def __init__(self, address): - super(UnverifiedAddressError, self).__init__() - self.address = address - - def __str__(self): - return self.address - class IUser(Interface): """A basic user.""" - real_name = Attribute( - """This user's real name.""") + display_name = Attribute( + """This user's display name.""") password = Attribute( """This user's password information.""") @@ -68,17 +61,17 @@ class IUser(Interface): memberships = Attribute( """A roster of this user's memberships.""") - def register(email, real_name=None): + def register(email, display_name=None): """Register the given email address and link it to this user. :param email: The text email address to register. :type email: str - :param real_name: The user's real name. If not given the empty string - is used. - :type real_name: str + :param display_name: The user's display name. If not given the empty + string is used. + :type display_name: str :return: The address object linked to the user. If the associated address object already existed (unlinked to a user) then the - `real_name` parameter is ignored. + `display_name` parameter is ignored. :rtype: `IAddress` :raises AddressAlreadyLinkedError: if this `IAddress` is already linked to another user. diff --git a/src/mailman/interfaces/usermanager.py b/src/mailman/interfaces/usermanager.py index 364fdc6e7..773bd1046 100644 --- a/src/mailman/interfaces/usermanager.py +++ b/src/mailman/interfaces/usermanager.py @@ -32,13 +32,13 @@ from zope.interface import Interface, Attribute class IUserManager(Interface): """The global user management service.""" - def create_user(email=None, real_name=None): + def create_user(email=None, display_name=None): """Create and return an `IUser`. :param email: The text email address for the user being created. :type email: str - :param real_name: The real name of the user. - :type real_name: str + :param display_name: The display name of the user. + :type display_name: str :return: The newly created user, with the given email address and real name, if given. :rtype: `IUser` @@ -74,15 +74,15 @@ class IUserManager(Interface): users = Attribute( """An iterator over all the `IUsers` managed by this user manager.""") - def create_address(email, real_name=None): + def create_address(email, display_name=None): """Create and return an address unlinked to any user. :param email: The text email address for the address being created. :type email: str - :param real_name: The real name associated with the address. - :type real_name: str + :param display_name: The display name associated with the address. + :type display_name: str :return: The newly created address object, with the given email - address and real name, if given. + address and display name, if given. :rtype: `IAddress` :raises ExistingAddressError: when the email address is already registered. diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 1a891a7c7..a0a13a16b 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -41,7 +41,7 @@ class Address(Model): id = Int(primary=True) email = Unicode() _original = Unicode() - real_name = Unicode() + display_name = Unicode() verified_on = DateTime() registered_on = DateTime() @@ -50,17 +50,17 @@ class Address(Model): preferences_id = Int() preferences = Reference(preferences_id, 'Preferences.id') - def __init__(self, email, real_name): + def __init__(self, email, display_name): super(Address, self).__init__() lower_case = email.lower() self.email = lower_case - self.real_name = real_name + self.display_name = display_name self._original = (None if lower_case == email else email) self.registered_on = now() def __str__(self): addr = (self.email if self._original is None else self._original) - return formataddr((self.real_name, addr)) + return formataddr((self.display_name, addr)) def __repr__(self): verified = ('verified' if self.verified_on else 'not verified') diff --git a/src/mailman/model/docs/addresses.rst b/src/mailman/model/docs/addresses.rst index 01e68c954..dfeac2b2a 100644 --- a/src/mailman/model/docs/addresses.rst +++ b/src/mailman/model/docs/addresses.rst @@ -29,7 +29,7 @@ Creating an unlinked email address is straightforward. However, such addresses have no real name. - >>> print address_1.real_name + >>> print address_1.display_name <BLANKLINE> You can also create an email address object with a real name. @@ -39,7 +39,7 @@ You can also create an email address object with a real name. >>> dump_list(address.email for address in user_manager.addresses) aperson@example.com bperson@example.com - >>> dump_list(address.real_name for address in user_manager.addresses) + >>> dump_list(address.display_name for address in user_manager.addresses) <BLANKLINE> Ben Person @@ -53,8 +53,8 @@ while the ``repr()`` carries more information. You can assign real names to existing addresses. - >>> address_1.real_name = 'Anne Person' - >>> dump_list(address.real_name for address in user_manager.addresses) + >>> address_1.display_name = 'Anne Person' + >>> dump_list(address.display_name for address in user_manager.addresses) Anne Person Ben Person @@ -77,7 +77,7 @@ interface. aperson@example.com bperson@example.com cperson@example.com - >>> dump_list(address.real_name for address in user_manager.addresses) + >>> dump_list(address.display_name for address in user_manager.addresses) Anne Person Ben Person Claire Person @@ -101,7 +101,7 @@ You can remove an unlinked address from the user manager. >>> dump_list(address.email for address in user_manager.addresses) bperson@example.com cperson@example.com - >>> dump_list(address.real_name for address in user_manager.addresses) + >>> dump_list(address.display_name for address in user_manager.addresses) Ben Person Claire Person diff --git a/src/mailman/model/docs/pending.rst b/src/mailman/model/docs/pending.rst index 707e8a7fc..1bf1ee0e9 100644 --- a/src/mailman/model/docs/pending.rst +++ b/src/mailman/model/docs/pending.rst @@ -23,7 +23,7 @@ token that can be used in urls and such. >>> subscription = SimplePendable( ... type='subscription', ... address='aperson@example.com', - ... realname='Anne Person', + ... display_name='Anne Person', ... language='en', ... password='xyz') >>> token = pendingdb.add(subscription) @@ -40,11 +40,11 @@ is returned. None >>> pendable = pendingdb.confirm(token) >>> dump_msgdata(pendable) - address : aperson@example.com - language: en - password: xyz - realname: Anne Person - type : subscription + address : aperson@example.com + display_name: Anne Person + language : en + password : xyz + type : subscription After confirmation, the token is no longer in the database. diff --git a/src/mailman/model/docs/registration.rst b/src/mailman/model/docs/registration.rst index 5ed2503b3..eecb3a8cd 100644 --- a/src/mailman/model/docs/registration.rst +++ b/src/mailman/model/docs/registration.rst @@ -102,9 +102,9 @@ But this address is waiting for confirmation. >>> dump_msgdata(pendingdb.confirm(token, expunge=False)) delivery_mode: regular + display_name : Anne Person email : aperson@example.com list_name : alpha@example.com - real_name : Anne Person type : registration @@ -275,12 +275,13 @@ different except that the new address will still need to be verified before it can be used. :: + >>> from mailman.utilities.datetime import now >>> dperson = user_manager.create_user( ... 'dperson@example.com', 'Dave Person') >>> dperson <User "Dave Person" (...) at ...> >>> address = user_manager.get_address('dperson@example.com') - >>> address.verified_on = datetime.now() + >>> address.verified_on = now() >>> from operator import attrgetter >>> dump_list(repr(address) for address in dperson.addresses) diff --git a/src/mailman/model/docs/requests.rst b/src/mailman/model/docs/requests.rst index 31597cf3a..a20823a91 100644 --- a/src/mailman/model/docs/requests.rst +++ b/src/mailman/model/docs/requests.rst @@ -205,7 +205,7 @@ For this section, we need a mailing list and at least one message. >>> mlist = create_list('alist@example.com') >>> mlist.preferred_language = 'en' - >>> mlist.real_name = 'A Test List' + >>> mlist.display_name = 'A Test List' >>> msg = message_from_string("""\ ... From: aperson@example.org ... To: alist@example.com @@ -230,8 +230,7 @@ We can also hold a message with some additional metadata. # collisions in the message storage. >>> del msg['message-id'] >>> msgdata = dict(sender='aperson@example.com', - ... approved=True, - ... received_time=123.45) + ... approved=True) >>> id_2 = moderator.hold_message(mlist, msg, msgdata, 'Feeling ornery') >>> requests.get_request(id_2) is not None True @@ -453,8 +452,7 @@ queue when the message is held. MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit - Subject: New subscription request to list A Test List from - cperson@example.org + Subject: New subscription request to A Test List from cperson@example.org From: alist-owner@example.com To: alist-owner@example.com Message-ID: ... @@ -563,8 +561,7 @@ subscription and the fact that they may need to approve it. MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit - Subject: New subscription request to list A Test List from - fperson@example.org + Subject: New subscription request to A Test List from fperson@example.org From: alist-owner@example.com To: alist-owner@example.com Message-ID: ... @@ -666,7 +663,7 @@ The admin message is sent to the moderators. Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List subscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -678,7 +675,7 @@ The admin message is sent to the moderators. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) @@ -696,7 +693,7 @@ Frank Person is now a member of the mailing list. <Language [en] English (USA)> >>> print member.delivery_mode DeliveryMode.regular - >>> print member.user.real_name + >>> print member.user.display_name Frank Person >>> print member.user.password {CLEARTEXT}abcxyz @@ -890,7 +887,7 @@ The goodbye message... Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List unsubscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -901,7 +898,7 @@ The goodbye message... >>> dump_msgdata(messages[1].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) diff --git a/src/mailman/model/docs/usermanager.rst b/src/mailman/model/docs/usermanager.rst index 8ad8d71a4..727f82835 100644 --- a/src/mailman/model/docs/usermanager.rst +++ b/src/mailman/model/docs/usermanager.rst @@ -26,7 +26,7 @@ have a password. >>> dump_list(address.email for address in user.addresses) *Empty* - >>> print user.real_name + >>> print user.display_name <BLANKLINE> >>> print user.password None @@ -38,8 +38,8 @@ The user has preferences, but none of them will be specified. A user can be assigned a real name. - >>> user.real_name = 'Anne Person' - >>> dump_list(user.real_name for user in user_manager.users) + >>> user.display_name = 'Anne Person' + >>> dump_list(user.display_name for user in user_manager.users) Anne Person A user can be assigned a password. @@ -55,25 +55,25 @@ You can also create a user with an address to start out with. True >>> dump_list(address.email for address in user_2.addresses) bperson@example.com - >>> dump_list(user.real_name for user in user_manager.users) + >>> dump_list(user.display_name for user in user_manager.users) <BLANKLINE> Anne Person As above, you can assign a real name to such users. - >>> user_2.real_name = 'Ben Person' - >>> dump_list(user.real_name for user in user_manager.users) + >>> user_2.display_name = 'Ben Person' + >>> dump_list(user.display_name for user in user_manager.users) Anne Person Ben Person You can also create a user with just a real name. - >>> user_3 = user_manager.create_user(real_name='Claire Person') + >>> user_3 = user_manager.create_user(display_name='Claire Person') >>> verifyObject(IUser, user_3) True >>> dump_list(address.email for address in user.addresses) *Empty* - >>> dump_list(user.real_name for user in user_manager.users) + >>> dump_list(user.display_name for user in user_manager.users) Anne Person Ben Person Claire Person @@ -85,9 +85,9 @@ Finally, you can create a user with both an address and a real name. True >>> dump_list(address.email for address in user_4.addresses) dperson@example.com - >>> dump_list(address.real_name for address in user_4.addresses) + >>> dump_list(address.display_name for address in user_4.addresses) Dan Person - >>> dump_list(user.real_name for user in user_manager.users) + >>> dump_list(user.display_name for user in user_manager.users) Anne Person Ben Person Claire Person @@ -101,7 +101,7 @@ You delete users by going through the user manager. The deleted user is no longer available through the user manager iterator. >>> user_manager.delete_user(user) - >>> dump_list(user.real_name for user in user_manager.users) + >>> dump_list(user.display_name for user in user_manager.users) Ben Person Claire Person Dan Person diff --git a/src/mailman/model/docs/users.rst b/src/mailman/model/docs/users.rst index 7325d66b9..ae2acfd48 100644 --- a/src/mailman/model/docs/users.rst +++ b/src/mailman/model/docs/users.rst @@ -21,17 +21,17 @@ Users may have a real name and a password. >>> user_1 = user_manager.create_user() >>> user_1.password = b'my password' - >>> user_1.real_name = 'Zoe Person' - >>> dump_list(user.real_name for user in user_manager.users) + >>> user_1.display_name = 'Zoe Person' + >>> dump_list(user.display_name for user in user_manager.users) Zoe Person >>> dump_list(user.password for user in user_manager.users) my password The password and real name can be changed at any time. - >>> user_1.real_name = 'Zoe X. Person' + >>> user_1.display_name = 'Zoe X. Person' >>> user_1.password = b'another password' - >>> dump_list(user.real_name for user in user_manager.users) + >>> dump_list(user.display_name for user in user_manager.users) Zoe X. Person >>> dump_list(user.password for user in user_manager.users) another password @@ -78,7 +78,7 @@ address on a user object. >>> dump_list(address.email for address in user_1.addresses) zperson@example.com zperson@example.org - >>> dump_list(address.real_name for address in user_1.addresses) + >>> dump_list(address.display_name for address in user_1.addresses) <BLANKLINE> Zoe Person @@ -90,7 +90,7 @@ You can also create the address separately and then link it to the user. zperson@example.com zperson@example.net zperson@example.org - >>> dump_list(address.real_name for address in user_1.addresses) + >>> dump_list(address.display_name for address in user_1.addresses) <BLANKLINE> <BLANKLINE> Zoe Person diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index cb56a36b6..0ea87a082 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -25,8 +25,6 @@ __all__ = [ ] -import datetime - from zope.event import notify from zope.interface import implements @@ -36,6 +34,7 @@ from mailman.interfaces.listmanager import ( IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent, ListDeletedEvent, ListDeletingEvent) from mailman.model.mailinglist import MailingList +from mailman.utilities.datetime import now @@ -57,7 +56,7 @@ class ListManager: if mlist: raise ListAlreadyExistsError(fqdn_listname) mlist = MailingList(fqdn_listname) - mlist.created_at = datetime.datetime.now() + mlist.created_at = now() config.db.store.add(mlist) notify(ListCreatedEvent(mlist)) return mlist diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 4a6b000ec..e397d59d6 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -38,7 +38,7 @@ from zope.interface import implements from mailman.config import config from mailman.database.model import Model from mailman.database.types import Enum -from mailman.interfaces.action import Action +from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.address import IAddress from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition @@ -115,6 +115,7 @@ class MailingList(Model): autorespond_requests = Enum(ResponseAction) autoresponse_request_text = Unicode() # Content filters. + filter_action = Enum(FilterAction) filter_content = Bool() collapse_alternatives = Bool() convert_html_to_plaintext = Bool() @@ -167,13 +168,15 @@ class MailingList(Model): nondigestable = Bool() nonmember_rejection_notice = Unicode() obscure_addresses = Bool() + owner_chain = Unicode() + owner_pipeline = Unicode() personalize = Enum(Personalization) post_id = Int() posting_chain = Unicode() posting_pipeline = Unicode() _preferred_language = Unicode(name='preferred_language') private_roster = Bool() - real_name = Unicode() + display_name = Unicode() reject_these_nonmembers = Pickle() reply_goes_to_list = Enum(ReplyToMunging) reply_to_address = Unicode() @@ -206,7 +209,7 @@ class MailingList(Model): # rosters explicitly. self.__storm_loaded__() self.personalize = Personalization.none - self.real_name = string.capwords( + self.display_name = string.capwords( SPACE.join(listname.split(UNDERSCORE))) makedirs(self.data_path) diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 72746295b..557361c6f 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -29,7 +29,6 @@ __all__ = [ import time import random import hashlib -import datetime from lazr.config import as_timedelta from storm.locals import DateTime, Int, RawStr, ReferenceSet, Unicode @@ -40,6 +39,7 @@ from mailman.config import config from mailman.database.model import Model from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) +from mailman.utilities.datetime import now from mailman.utilities.modules import call_name @@ -98,8 +98,8 @@ class Pendings: # does the hash calculation. The integral parts of the time values # are discarded because they're the most predictable bits. for attempts in range(3): - now = time.time() - x = random.random() + now % 1.0 + time.clock() % 1.0 + right_now = time.time() + x = random.random() + right_now % 1.0 + time.clock() % 1.0 # Use sha1 because it produces shorter strings. token = hashlib.sha1(repr(x)).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal @@ -111,7 +111,7 @@ class Pendings: # Create the record, and then the individual key/value pairs. pending = Pended( token=token, - expiration_date=datetime.datetime.now() + lifetime) + expiration_date=now() + lifetime) for key, value in pendable.items(): if isinstance(key, str): key = unicode(key, 'utf-8') @@ -160,9 +160,9 @@ class Pendings: def evict(self): store = config.db.store - now = datetime.datetime.now() + right_now = now() for pending in store.find(Pended): - if pending.expiration_date < now: + if pending.expiration_date < right_now: # Find all PendedKeyValue entries that are associated with the # pending object's ID. q = store.find(PendedKeyValue, diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index 35ddcf438..48d434ab1 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -64,16 +64,24 @@ class AbstractRoster: def __init__(self, mlist): self._mlist = mlist + def _query(self): + return config.db.store.find( + Member, + mailing_list=self._mlist.fqdn_listname, + role=self.role) + @property def members(self): """See `IRoster`.""" - for member in config.db.store.find( - Member, - mailing_list=self._mlist.fqdn_listname, - role=self.role): + for member in self._query(): yield member @property + def member_count(self): + """See `IRoster`.""" + return self._query().count() + + @property def users(self): """See `IRoster`.""" # Members are linked to addresses, which in turn are linked to users. @@ -149,18 +157,12 @@ class AdministratorRoster(AbstractRoster): name = 'administrator' - @property - def members(self): - """See `IRoster`.""" - # Administrators are defined as the union of the owners and the - # moderators. - members = config.db.store.find( - Member, - Member.mailing_list == self._mlist.fqdn_listname, - Or(Member.role == MemberRole.owner, - Member.role == MemberRole.moderator)) - for member in members: - yield member + def _query(self): + return config.db.store.find( + Member, + Member.mailing_list == self._mlist.fqdn_listname, + Or(Member.role == MemberRole.owner, + Member.role == MemberRole.moderator)) def get_member(self, address): """See `IRoster`.""" @@ -184,6 +186,14 @@ class AdministratorRoster(AbstractRoster): class DeliveryMemberRoster(AbstractRoster): """Return all the members having a particular kind of delivery.""" + @property + def member_count(self): + """See `IRoster`.""" + # XXX 2012-03-15 BAW: It would be nice to make this more efficient. + # The problem is that you'd have to change the loop in _get_members() + # checking the delivery mode to a query parameter. + return len(tuple(self.members)) + def _get_members(self, *delivery_modes): """The set of members for a mailing list, filter by delivery mode. @@ -234,13 +244,10 @@ class Subscribers(AbstractRoster): name = 'subscribers' - @property - def members(self): - """See `IRoster`.""" - for member in config.db.store.find( - Member, - mailing_list=self._mlist.fqdn_listname): - yield member + def _query(self): + return config.db.store.find( + Member, + mailing_list=self._mlist.fqdn_listname) @@ -254,15 +261,23 @@ class Memberships: def __init__(self, user): self._user = user - @property - def members(self): - """See `IRoster`.""" + def _query(self): results = config.db.store.find( Member, Or(Member.user_id == self._user.id, And(Address.user_id == self._user.id, Member.address_id == Address.id))) - for member in results.config(distinct=True): + return results.config(distinct=True) + + @property + def member_count(self): + """See `IRoster`.""" + return self._query().count() + + @property + def members(self): + """See `IRoster`.""" + for member in self._query(): yield member @property diff --git a/src/mailman/model/tests/test_roster.py b/src/mailman/model/tests/test_roster.py new file mode 100644 index 000000000..8d5a7b81b --- /dev/null +++ b/src/mailman/model/tests/test_roster.py @@ -0,0 +1,156 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test rosters.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMailingListRoster', + 'TestMembershipsRoster', + ] + + +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.interfaces.member import DeliveryMode, MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import now + + + +class TestMailingListRoster(unittest.TestCase): + """Test various aspects of a mailing list's roster.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + user_manager = getUtility(IUserManager) + self._anne = user_manager.create_address('anne@example.com') + self._bart = user_manager.create_address('bart@example.com') + self._cris = user_manager.create_address('cris@example.com') + + def test_no_members(self): + # Nobody with any role is subscribed to the mailing list. + self.assertEqual(self._mlist.owners.member_count, 0) + self.assertEqual(self._mlist.moderators.member_count, 0) + self.assertEqual(self._mlist.administrators.member_count, 0) + self.assertEqual(self._mlist.members.member_count, 0) + self.assertEqual(self._mlist.regular_members.member_count, 0) + self.assertEqual(self._mlist.digest_members.member_count, 0) + self.assertEqual(self._mlist.subscribers.member_count, 0) + + def test_one_regular_member(self): + # One person getting regular delivery is subscribed to the mailing + # list as a member. + self._mlist.subscribe(self._anne, role=MemberRole.member) + self.assertEqual(self._mlist.owners.member_count, 0) + self.assertEqual(self._mlist.moderators.member_count, 0) + self.assertEqual(self._mlist.administrators.member_count, 0) + self.assertEqual(self._mlist.members.member_count, 1) + self.assertEqual(self._mlist.regular_members.member_count, 1) + self.assertEqual(self._mlist.digest_members.member_count, 0) + self.assertEqual(self._mlist.subscribers.member_count, 1) + + def test_two_regular_members(self): + # Two people getting regular delivery are subscribed to the mailing + # list as members. + self._mlist.subscribe(self._anne, role=MemberRole.member) + self._mlist.subscribe(self._bart, role=MemberRole.member) + self.assertEqual(self._mlist.owners.member_count, 0) + self.assertEqual(self._mlist.moderators.member_count, 0) + self.assertEqual(self._mlist.administrators.member_count, 0) + self.assertEqual(self._mlist.members.member_count, 2) + self.assertEqual(self._mlist.regular_members.member_count, 2) + self.assertEqual(self._mlist.digest_members.member_count, 0) + self.assertEqual(self._mlist.subscribers.member_count, 2) + + def test_one_regular_members_one_digest_member(self): + # Two people are subscribed to the mailing list as members. One gets + # regular delivery and one gets digest delivery. + self._mlist.subscribe(self._anne, role=MemberRole.member) + member = self._mlist.subscribe(self._bart, role=MemberRole.member) + member.preferences.delivery_mode = DeliveryMode.mime_digests + self.assertEqual(self._mlist.owners.member_count, 0) + self.assertEqual(self._mlist.moderators.member_count, 0) + self.assertEqual(self._mlist.administrators.member_count, 0) + self.assertEqual(self._mlist.members.member_count, 2) + self.assertEqual(self._mlist.regular_members.member_count, 1) + self.assertEqual(self._mlist.digest_members.member_count, 1) + self.assertEqual(self._mlist.subscribers.member_count, 2) + + def test_a_person_is_both_a_member_and_an_owner(self): + # Anne is the owner of a mailing list and she gets subscribed as a + # member of the mailing list, receiving regular deliveries. + self._mlist.subscribe(self._anne, role=MemberRole.member) + self._mlist.subscribe(self._anne, role=MemberRole.owner) + self.assertEqual(self._mlist.owners.member_count, 1) + self.assertEqual(self._mlist.moderators.member_count, 0) + self.assertEqual(self._mlist.administrators.member_count, 1) + self.assertEqual(self._mlist.members.member_count, 1) + self.assertEqual(self._mlist.regular_members.member_count, 1) + self.assertEqual(self._mlist.digest_members.member_count, 0) + self.assertEqual(self._mlist.subscribers.member_count, 2) + + def test_a_bunch_of_members_and_administrators(self): + # Anne is the owner of a mailing list, and Bart is a moderator. Anne + # gets subscribed as a member of the mailing list, receiving regular + # deliveries. Cris subscribes to the mailing list as a digest member. + self._mlist.subscribe(self._anne, role=MemberRole.owner) + self._mlist.subscribe(self._bart, role=MemberRole.moderator) + self._mlist.subscribe(self._anne, role=MemberRole.member) + member = self._mlist.subscribe(self._cris, role=MemberRole.member) + member.preferences.delivery_mode = DeliveryMode.mime_digests + self.assertEqual(self._mlist.owners.member_count, 1) + self.assertEqual(self._mlist.moderators.member_count, 1) + self.assertEqual(self._mlist.administrators.member_count, 2) + self.assertEqual(self._mlist.members.member_count, 2) + self.assertEqual(self._mlist.regular_members.member_count, 1) + self.assertEqual(self._mlist.digest_members.member_count, 1) + self.assertEqual(self._mlist.subscribers.member_count, 4) + + + +class TestMembershipsRoster(unittest.TestCase): + """Test the memberships roster.""" + + layer = ConfigLayer + + def setUp(self): + self._ant = create_list('ant@example.com') + self._bee = create_list('bee@example.com') + user_manager = getUtility(IUserManager) + self._anne = user_manager.create_user('anne@example.com') + preferred = list(self._anne.addresses)[0] + preferred.verified_on = now() + self._anne.preferred_address = preferred + + def test_no_memberships(self): + # An unsubscribed user has no memberships. + self.assertEqual(self._anne.memberships.member_count, 0) + + def test_subscriptions(self): + # Anne subscribes to a couple of mailing lists. + self._ant.subscribe(self._anne) + self._bee.subscribe(self._anne) + self.assertEqual(self._anne.memberships.member_count, 2) diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index d1f59f957..11c719eea 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -51,7 +51,7 @@ class User(Model): implements(IUser) id = Int(primary=True) - real_name = Unicode() + display_name = Unicode() password = RawStr() _user_id = UUID() _created_on = DateTime() @@ -62,20 +62,20 @@ class User(Model): preferences_id = Int() preferences = Reference(preferences_id, 'Preferences.id') - def __init__(self, real_name=None, preferences=None): + def __init__(self, display_name=None, preferences=None): super(User, self).__init__() self._created_on = date_factory.now() user_id = uid_factory.new_uid() assert config.db.store.find(User, _user_id=user_id).count() == 0, ( 'Duplicate user id {0}'.format(user_id)) self._user_id = user_id - self.real_name = ('' if real_name is None else real_name) + self.display_name = ('' if display_name is None else display_name) self.preferences = preferences config.db.store.add(self) def __repr__(self): short_user_id = self.user_id.int - return '<User "{0.real_name}" ({2}) at {1:#x}>'.format( + return '<User "{0.display_name}" ({2}) at {1:#x}>'.format( self, id(self), short_user_id) @property @@ -132,14 +132,14 @@ class User(Model): assert found.count() == 1, 'Unexpected count' return found[0].user is self - def register(self, email, real_name=None): + def register(self, email, display_name=None): """See `IUser`.""" # First, see if the address already exists address = config.db.store.find(Address, email=email).one() if address is None: - if real_name is None: - real_name = '' - address = Address(email=email, real_name=real_name) + if display_name is None: + display_name = '' + address = Address(email=email, display_name=display_name) address.preferences = Preferences() # Link the address to the user if it is not already linked. if address.user is not None: diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py index 0c5322987..c8a5c65a2 100644 --- a/src/mailman/model/usermanager.py +++ b/src/mailman/model/usermanager.py @@ -40,11 +40,11 @@ from mailman.model.user import User class UserManager: implements(IUserManager) - def create_user(self, email=None, real_name=None): + def create_user(self, email=None, display_name=None): """See `IUserManager`.""" - user = User(real_name, Preferences()) + user = User(display_name, Preferences()) if email: - address = self.create_address(email, real_name) + address = self.create_address(email, display_name) user.link(address) return user @@ -72,18 +72,18 @@ class UserManager: for user in config.db.store.find(User): yield user - def create_address(self, email, real_name=None): + def create_address(self, email, display_name=None): """See `IUserManager`.""" addresses = config.db.store.find(Address, email=email.lower()) if addresses.count() == 1: found = addresses[0] raise ExistingAddressError(found.original_email) assert addresses.count() == 0, 'Unexpected results' - if real_name is None: - real_name = '' + if display_name is None: + display_name = '' # It's okay not to lower case the 'email' argument because the # constructor will do the right thing. - address = Address(email, real_name) + address = Address(email, display_name) address.preferences = Preferences() config.db.store.add(address) return address diff --git a/src/mailman/mta/personalized.py b/src/mailman/mta/personalized.py index a0b0d4b76..cebc73cae 100644 --- a/src/mailman/mta/personalized.py +++ b/src/mailman/mta/personalized.py @@ -63,7 +63,7 @@ class PersonalizedMixin: # Convert the unicode name to an email-safe representation. # Create a Header instance for the name so that it's properly # encoded for email transport. - name = Header(user.real_name).encode() + name = Header(user.display_name).encode() msg.replace_header('To', formataddr((name, recipient))) diff --git a/src/mailman/mta/postfix.py b/src/mailman/mta/postfix.py index 14f19635a..32bdb8268 100644 --- a/src/mailman/mta/postfix.py +++ b/src/mailman/mta/postfix.py @@ -27,7 +27,6 @@ __all__ = [ import os import logging -import datetime from flufl.lock import Lock from operator import attrgetter @@ -38,6 +37,7 @@ from mailman.config import config from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mta import ( IMailTransportAgentAliases, IMailTransportAgentLifecycle) +from mailman.utilities.datetime import now log = logging.getLogger('mailman.error') @@ -124,7 +124,7 @@ class LMTP: # file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're # doing, and can keep the two files properly in sync. If you screw it up, # you're on your own. -""".format(datetime.datetime.now().replace(microsecond=0)) +""".format(now().replace(microsecond=0)) sort_key = attrgetter('list_name') for domain in sorted(by_domain): print >> fp, """\ diff --git a/src/mailman/pipeline/docs/scrubber.rst b/src/mailman/pipeline/docs/scrubber.rst deleted file mode 100644 index 86a8161a7..000000000 --- a/src/mailman/pipeline/docs/scrubber.rst +++ /dev/null @@ -1,230 +0,0 @@ -============ -The scrubber -============ - -The scrubber is an integral part of Mailman, both in the normal delivery of -messages and in components such as the archiver. Its primary purpose is to -scrub attachments from messages so that binary goop doesn't end up in an -archive message. - - >>> mlist = create_list('_xtest@example.com') - >>> mlist.preferred_language = 'en' - -Helper functions for getting the attachment data. -:: - - >>> import os, re - >>> def read_attachment(filename, remove=True): - ... path = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, - ... mlist.fqdn_listname, filename) - ... fp = open(path) - ... try: - ... data = fp.read() - ... finally: - ... fp.close() - ... if remove: - ... os.unlink(path) - ... return data - - >>> from urlparse import urlparse - >>> def read_url_from_message(msg): - ... url = None - ... for line in msg.get_payload().splitlines(): - ... mo = re.match('URL: <(?P<url>[^>]+)>', line) - ... if mo: - ... url = mo.group('url') - ... break - ... path = '/'.join(urlparse(url).path.split('/')[3:]) - ... return read_attachment(path) - - -Saving attachments -================== - -The Scrubber handler exposes a function called ``save_attachment()`` which can -be used to strip various types of attachments and store them in the archive -directory. This is a public interface used by components outside the normal -processing pipeline. - -Site administrators can decide whether the scrubber should use the attachment -filename suggested in the message's ``Content-Disposition:`` header or not. -If enabled, the filename will be used when this header attribute is present -(yes, this is an unfortunate double negative). -:: - - >>> config.push('test config', """ - ... [scrubber] - ... use_attachment_filename: yes - ... """) - >>> msg = message_from_string("""\ - ... Content-Type: image/gif; name="xtest.gif" - ... Content-Transfer-Encoding: base64 - ... Content-Disposition: attachment; filename="xtest.gif" - ... - ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw== - ... """) - - >>> from mailman.pipeline.scrubber import save_attachment - >>> print save_attachment(mlist, msg, 'dir') - <http://www.example.com/pipermail/_xtest@example.com/dir/xtest.gif> - >>> data = read_attachment('dir/xtest.gif') - >>> print data[:6] - GIF87a - >>> len(data) - 34 - -Saving the attachment does not alter the original message. - - >>> print msg.as_string() - Content-Type: image/gif; name="xtest.gif" - Content-Transfer-Encoding: base64 - Content-Disposition: attachment; filename="xtest.gif" - <BLANKLINE> - R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw== - -The site administrator can also configure Mailman to ignore the -``Content-Disposition:`` filename. This is the default. - - >>> config.pop('test config') - >>> config.push('test config', """ - ... [scrubber] - ... use_attachment_filename: no - ... """) - >>> msg = message_from_string("""\ - ... Content-Type: image/gif; name="xtest.gif" - ... Content-Transfer-Encoding: base64 - ... Content-Disposition: attachment; filename="xtest.gif" - ... - ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw== - ... """) - >>> print save_attachment(mlist, msg, 'dir') - <http://www.example.com/pipermail/_xtest@example.com/dir/attachment.gif> - >>> data = read_attachment('dir/xtest.gif') - Traceback (most recent call last): - IOError: [Errno ...] No such file or directory: - u'.../archives/private/_xtest@example.com/dir/xtest.gif' - >>> data = read_attachment('dir/attachment.gif') - >>> print data[:6] - GIF87a - >>> len(data) - 34 - - -Scrubbing image attachments -=========================== - -When scrubbing image attachments, the original message is modified to include -a reference to the attachment file as available through the on-line archive. - - >>> msg = message_from_string("""\ - ... MIME-Version: 1.0 - ... Content-Type: multipart/mixed; boundary="BOUNDARY" - ... - ... --BOUNDARY - ... Content-type: text/plain; charset=us-ascii - ... - ... This is a message. - ... --BOUNDARY - ... Content-Type: image/gif; name="xtest.gif" - ... Content-Transfer-Encoding: base64 - ... Content-Disposition: attachment; filename="xtest.gif" - ... - ... R0lGODdhAQABAIAAAAAAAAAAACwAAAAAAQABAAACAQUAOw== - ... --BOUNDARY-- - ... """) - >>> msgdata = {} - -The ``Scrubber.process()`` function is different than other handler process -functions in that it returns the scrubbed message. - - >>> from mailman.pipeline.scrubber import process - >>> scrubbed_msg = process(mlist, msg, msgdata) - >>> scrubbed_msg is msg - True - >>> print scrubbed_msg.as_string() - MIME-Version: 1.0 - Message-ID: ... - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - <BLANKLINE> - This is a message. - -------------- next part -------------- - A non-text attachment was scrubbed... - Name: xtest.gif - Type: image/gif - Size: 34 bytes - Desc: not available - URL: <http://www.example.com/pipermail/_xtest@example.com/attachments/.../attachment.gif> - <BLANKLINE> - -This is the same as the transformed message originally passed in. - - >>> print msg.as_string() - MIME-Version: 1.0 - Message-ID: ... - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - <BLANKLINE> - This is a message. - -------------- next part -------------- - A non-text attachment was scrubbed... - Name: xtest.gif - Type: image/gif - Size: 34 bytes - Desc: not available - URL: <http://www.example.com/pipermail/_xtest@example.com/attachments/.../attachment.gif> - <BLANKLINE> - >>> msgdata - {} - -The URL will point to the attachment sitting in the archive. - - >>> data = read_url_from_message(msg) - >>> data[:6] - 'GIF87a' - >>> len(data) - 34 - - -Scrubbing text attachments -========================== - -Similar to image attachments, text attachments will also be scrubbed, but the -placeholder will be slightly different. - - >>> msg = message_from_string("""\ - ... MIME-Version: 1.0 - ... Content-Type: multipart/mixed; boundary="BOUNDARY" - ... - ... --BOUNDARY - ... Content-type: text/plain; charset=us-ascii; format=flowed; delsp=no - ... - ... This is a message. - ... --BOUNDARY - ... Content-type: text/plain; name="xtext.txt" - ... Content-Disposition: attachment; filename="xtext.txt" - ... - ... This is a text attachment. - ... --BOUNDARY-- - ... """) - >>> scrubbed_msg = process(mlist, msg, {}) - >>> print scrubbed_msg.as_string() - MIME-Version: 1.0 - Message-ID: ... - Content-Transfer-Encoding: 7bit - Content-Type: text/plain; charset="us-ascii"; format="flowed"; delsp="no" - <BLANKLINE> - This is a message. - -------------- next part -------------- - An embedded and charset-unspecified text was scrubbed... - Name: xtext.txt - URL: <http://www.example.com/pipermail/_xtest@example.com/attachments/.../attachment.txt> - <BLANKLINE> - >>> read_url_from_message(msg) - 'This is a text attachment.' - - -Clean up -======== - - >>> config.pop('test config') diff --git a/src/mailman/pipeline/owner_recipients.py b/src/mailman/pipeline/owner_recipients.py deleted file mode 100644 index 9e4bbf174..000000000 --- a/src/mailman/pipeline/owner_recipients.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Calculate the list owner recipients (includes moderators).""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'process', - ] - - - -def process(mlist, msg, msgdata): - """Add owner recipients.""" - # The recipients are the owner and the moderator - msgdata['recipients'] = mlist.owner + mlist.moderator - # Don't decorate these messages with the header/footers - msgdata['nodecorate'] = True - msgdata['personalize'] = False diff --git a/src/mailman/pipeline/scrubber.py b/src/mailman/pipeline/scrubber.py deleted file mode 100644 index 0584c0a2c..000000000 --- a/src/mailman/pipeline/scrubber.py +++ /dev/null @@ -1,499 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Cleanse a message for archiving.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Scrubber', - ] - - -import os -import re -import time -import hashlib -import logging -import binascii - -from email.charset import Charset -from email.utils import make_msgid, parsedate -from flufl.lock import Lock -from lazr.config import as_boolean -from mimetypes import guess_all_extensions -from string import Template -from zope.interface import implements - -from mailman.config import config -from mailman.core.errors import DiscardMessage -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.utilities.filesystem import makedirs -from mailman.utilities.modules import find_name -from mailman.utilities.string import oneline, websafe - - -# Path characters for common platforms -pre = re.compile(r'[/\\:]') -# All other characters to strip out of Content-Disposition: filenames -# (essentially anything that isn't an alphanum, dot, dash, or underscore). -sre = re.compile(r'[^-\w.]') -# Regexp to strip out leading dots -dre = re.compile(r'^\.*') - -BR = '<br>\n' -SPACE = ' ' - -log = logging.getLogger('mailman.error') - - - -def guess_extension(ctype, ext): - """Find the extension mapped to the given content-type. - - mimetypes maps multiple extensions to the same type, e.g. .doc, .dot, and - .wiz are all mapped to application/msword. This sucks for finding the - best reverse mapping. If the extension is one of the giving mappings, - we'll trust that, otherwise we'll just guess. :/ - """ - all_extensions = guess_all_extensions(ctype, strict=False) - if ext in all_extensions: - return ext - return (all_extensions[0] if len(all_extensions) > 0 else None) - - - -def safe_strftime(fmt, t): - """A time.strftime() that eats exceptions, returning None instead.""" - try: - return time.strftime(fmt, t) - except (TypeError, ValueError, OverflowError): - return None - - -def calculate_attachments_dir(msg, msgdata): - """Calculate the directory for attachements. - - Calculate the directory that attachments for this message will go under. - To avoid inode limitations, the scheme will be: - archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files> - Start by calculating the date-based and msgid-hash components. - """ - fmt = '%Y%m%d' - datestr = msg.get('Date') - if datestr: - now = parsedate(datestr) - else: - now = time.gmtime(msgdata.get('received_time', time.time())) - datedir = safe_strftime(fmt, now) - if not datedir: - datestr = msgdata.get('X-List-Received-Date') - if datestr: - datedir = safe_strftime(fmt, datestr) - if not datedir: - # What next? Unixfrom, I guess. - parts = msg.get_unixfrom().split() - try: - month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, - 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12, - }.get(parts[3], 0) - day = int(parts[4]) - year = int(parts[6]) - except (IndexError, ValueError): - # Best we can do I think - month = day = year = 0 - datedir = '%04d%02d%02d' % (year, month, day) - assert datedir - # As for the msgid hash, we'll base this part on the Message-ID: so that - # all attachments for the same message end up in the same directory (we'll - # uniquify the filenames in that directory as needed). We use the first 2 - # and last 2 bytes of the SHA1 hash of the message id as the basis of the - # directory name. Clashes here don't really matter too much, and that - # still gives us a 32-bit space to work with. - msgid = msg['message-id'] - if msgid is None: - msgid = msg['Message-ID'] = make_msgid() - # We assume that the message id actually /is/ unique! - digest = hashlib.sha1(msgid).hexdigest() - return os.path.join('attachments', datedir, digest[:4] + digest[-4:]) - - -def replace_payload_by_text(msg, text, charset): - """Replace the payload of the message with some text.""" - # TK: This is a common function in replacing the attachment and the main - # message by a text (scrubbing). - del msg['content-type'] - del msg['content-transfer-encoding'] - if isinstance(text, unicode): - text = text.encode(charset) - if not isinstance(charset, str): - charset = str(charset) - msg.set_payload(text, charset) - - - -def process(mlist, msg, msgdata=None): - """Process the message through the scrubber.""" - sanitize = int(config.scrubber.archive_html_sanitizer) - outer = True - if msgdata is None: - msgdata = {} - if msgdata: - # msgdata is available if it is in GLOBAL_PIPELINE - # ie. not in digest or archiver - # check if the list owner want to scrub regular delivery - if not mlist.scrub_nondigest: - return - attachments_dir = calculate_attachments_dir(msg, msgdata) - charset = format_param = delsp = None - lcset = mlist.preferred_language.charset - lcset_out = Charset(lcset).output_charset or lcset - # Now walk over all subparts of this message and scrub out various types - for part in msg.walk(): - ctype = part.get_content_type() - # If the part is text/plain, we leave it alone - if ctype == 'text/plain': - # We need to choose a charset for the scrubbed message, so we'll - # arbitrarily pick the charset of the first text/plain part in the - # message. - # - # Also get the RFC 3676 stuff from this part. This seems to - # work okay for scrub_nondigest. It will also work as far as - # scrubbing messages for the archive is concerned, but Pipermail - # doesn't pay any attention to the RFC 3676 parameters. The plain - # format digest is going to be a disaster in any case as some of - # messages will be format="flowed" and some not. ToDigest creates - # its own Content-Type: header for the plain digest which won't - # have RFC 3676 parameters. If the message Content-Type: headers - # are retained for display in the digest, the parameters will be - # there for information, but not for the MUA. This is the best we - # can do without having get_payload() process the parameters. - if charset is None: - charset = part.get_content_charset(lcset) - format_param = part.get_param('format') - delsp = part.get_param('delsp') - # TK: if part is attached then check charset and scrub if none - if part.get('content-disposition') and \ - not part.get_content_charset(): - url = save_attachment(mlist, part, attachments_dir) - filename = part.get_filename(_('not available')) - filename = oneline(filename, lcset) - replace_payload_by_text(part, _("""\ -An embedded and charset-unspecified text was scrubbed... -Name: $filename -URL: $url -"""), lcset) - elif ctype == 'text/html' and isinstance(sanitize, int): - if sanitize == 0: - if outer: - raise DiscardMessage - replace_payload_by_text(part, - _('HTML attachment scrubbed and removed'), - # Adding charset arg and removing content-type - # sets content-type to text/plain - lcset) - elif sanitize == 2: - # By leaving it alone, Pipermail will automatically escape it - pass - elif sanitize == 3: - # Pull it out as an attachment but leave it unescaped. This - # is dangerous, but perhaps useful for heavily moderated - # lists. - url = save_attachment(mlist, part, attachments_dir, - filter_html=False) - replace_payload_by_text(part, _("""\ -An HTML attachment was scrubbed... -URL: $url -"""), lcset) - else: - # HTML-escape it and store it as an attachment, but make it - # look a /little/ bit prettier. :( - payload = websafe(part.get_payload(decode=True)) - # For whitespace in the margin, change spaces into - # non-breaking spaces, and tabs into 8 of those. Then use a - # mono-space font. Still looks hideous to me, but then I'd - # just as soon discard them. - lines = [s.replace(' ', ' ').replace('\t', ' ' * 8) - for s in payload.split('\n')] - payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n' - part.set_payload(payload) - # We're replacing the payload with the decoded payload so this - # will just get in the way. - del part['content-transfer-encoding'] - url = save_attachment(mlist, part, attachments_dir, - filter_html=False) - replace_payload_by_text(part, _("""\ -An HTML attachment was scrubbed... -URL: $url -"""), lcset) - elif ctype == 'message/rfc822': - # This part contains a submessage, so it too needs scrubbing - submsg = part.get_payload(0) - url = save_attachment(mlist, part, attachments_dir) - subject = submsg.get('subject', _('no subject')) - date = submsg.get('date', _('no date')) - who = submsg.get('from', _('unknown sender')) - size = len(str(submsg)) - replace_payload_by_text(part, _("""\ -An embedded message was scrubbed... -From: $who -Subject: $subject -Date: $date -Size: $size -URL: $url -"""), lcset) - # If the message isn't a multipart, then we'll strip it out as an - # attachment that would have to be separately downloaded. Pipermail - # will transform the url into a hyperlink. - elif part._payload and not part.is_multipart(): - payload = part.get_payload(decode=True) - ctype = part.get_content_type() - # XXX Under email 2.5, it is possible that payload will be None. - # This can happen when you have a Content-Type: multipart/* with - # only one part and that part has two blank lines between the - # first boundary and the end boundary. In email 3.0 you end up - # with a string in the payload. I think in this case it's safe to - # ignore the part. - if payload is None: - continue - size = len(payload) - url = save_attachment(mlist, part, attachments_dir) - desc = part.get('content-description', _('not available')) - desc = oneline(desc, lcset) - filename = part.get_filename(_('not available')) - filename = oneline(filename, lcset) - replace_payload_by_text(part, _("""\ -A non-text attachment was scrubbed... -Name: $filename -Type: $ctype -Size: $size bytes -Desc: $desc -URL: $url -"""), lcset) - outer = False - # We still have to sanitize multipart messages to flat text because - # Pipermail can't handle messages with list payloads. This is a kludge; - # def (n) clever hack ;). - if msg.is_multipart() and sanitize != 2: - # By default we take the charset of the first text/plain part in the - # message, but if there was none, we'll use the list's preferred - # language's charset. - if not charset or charset == 'us-ascii': - charset = lcset_out - else: - # normalize to the output charset if input/output are different - charset = Charset(charset).output_charset or charset - # We now want to concatenate all the parts which have been scrubbed to - # text/plain, into a single text/plain payload. We need to make sure - # all the characters in the concatenated string are in the same - # encoding, so we'll use the 'replace' key in the coercion call. - # BAW: Martin's original patch suggested we might want to try - # generalizing to utf-8, and that's probably a good idea (eventually). - text = [] - charsets = [] - for part in msg.walk(): - # TK: bug-id 1099138 and multipart - # MAS test payload - if part may fail if there are no headers. - if not part._payload or part.is_multipart(): - continue - # All parts should be scrubbed to text/plain by now. - partctype = part.get_content_type() - if partctype != 'text/plain': - text.append(_('Skipped content of type $partctype\n')) - continue - try: - t = part.get_payload(decode=True) or '' - # MAS: TypeError exception can occur if payload is None. This - # was observed with a message that contained an attached - # message/delivery-status part. Because of the special parsing - # of this type, this resulted in a text/plain sub-part with a - # null body. See bug 1430236. - except (binascii.Error, TypeError): - t = part.get_payload() or '' - # Email problem was solved by Mark Sapiro. (TK) - partcharset = part.get_content_charset('us-ascii') - try: - t = unicode(t, partcharset, 'replace') - except (UnicodeError, LookupError, ValueError, TypeError, - AssertionError): - # We can get here if partcharset is bogus in come way. - # Replace funny characters. We use errors='replace'. - t = unicode(t, 'ascii', 'replace') - # Separation is useful - if isinstance(t, basestring): - if not t.endswith('\n'): - t += '\n' - text.append(t) - if partcharset not in charsets: - charsets.append(partcharset) - # Now join the text and set the payload - sep = _('-------------- next part --------------\n') - assert isinstance(sep, unicode), ( - 'Expected a unicode separator, got %s' % type(sep)) - rept = sep.join(text) - # Replace entire message with text and scrubbed notice. - # Try with message charsets and utf-8 - if 'utf-8' not in charsets: - charsets.append('utf-8') - for charset in charsets: - try: - replace_payload_by_text(msg, rept, charset) - break - # Bogus charset can throw several exceptions - except (UnicodeError, LookupError, ValueError, TypeError, - AssertionError): - pass - if format_param: - msg.set_param('format', format_param) - if delsp: - msg.set_param('delsp', delsp) - return msg - - - -def save_attachment(mlist, msg, attachments_dir, filter_html=True): - fsdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, - mlist.fqdn_listname, attachments_dir) - makedirs(fsdir) - # Figure out the attachment type and get the decoded data - decodedpayload = msg.get_payload(decode=True) - # BAW: mimetypes ought to handle non-standard, but commonly found types, - # e.g. image/jpg (should be image/jpeg). For now we just store such - # things as application/octet-streams since that seems the safest. - ctype = msg.get_content_type() - # i18n file name is encoded - lcset = mlist.preferred_language.charset - filename = oneline(msg.get_filename(''), lcset) - filename, fnext = os.path.splitext(filename) - # For safety, we should confirm this is valid ext for content-type - # but we can use fnext if we introduce fnext filtering - if as_boolean(config.scrubber.use_attachment_filename_extension): - # HTML message doesn't have filename :-( - ext = fnext or guess_extension(ctype, fnext) - else: - ext = guess_extension(ctype, fnext) - if not ext: - # We don't know what it is, so assume it's just a shapeless - # application/octet-stream, unless the Content-Type: is - # message/rfc822, in which case we know we'll coerce the type to - # text/plain below. - if ctype == 'message/rfc822': - ext = '.txt' - else: - ext = '.bin' - # Allow only alphanumerics, dash, underscore, and dot - ext = sre.sub('', ext) - path = None - # We need a lock to calculate the next attachment number - with Lock(os.path.join(fsdir, 'attachments.lock')): - # Now base the filename on what's in the attachment, uniquifying it if - # necessary. - if (not filename or - not as_boolean(config.scrubber.use_attachment_filename)): - filebase = 'attachment' - else: - # Sanitize the filename given in the message headers - parts = pre.split(filename) - filename = parts[-1] - # Strip off leading dots - filename = dre.sub('', filename) - # Allow only alphanumerics, dash, underscore, and dot - filename = sre.sub('', filename) - # If the filename's extension doesn't match the type we guessed, - # which one should we go with? For now, let's go with the one we - # guessed so attachments can't lie about their type. Also, if the - # filename /has/ no extension, then tack on the one we guessed. - # The extension was removed from the name above. - filebase = filename - # Now we're looking for a unique name for this file on the file - # system. If msgdir/filebase.ext isn't unique, we'll add a counter - # after filebase, e.g. msgdir/filebase-cnt.ext - counter = 0 - extra = '' - while True: - path = os.path.join(fsdir, filebase + extra + ext) - # Generally it is not a good idea to test for file existance - # before just trying to create it, but the alternatives aren't - # wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't - # NFS-safe). Besides, we have an exclusive lock now, so we're - # guaranteed that no other process will be racing with us. - if os.path.exists(path): - counter += 1 - extra = '-%04d' % counter - else: - break - # `path' now contains the unique filename for the attachment. There's - # just one more step we need to do. If the part is text/html and - # ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be - # here), then send the attachment through the filter program for - # sanitization - if filter_html and ctype == 'text/html': - base, ext = os.path.splitext(path) - tmppath = base + '-tmp' + ext - fp = open(tmppath, 'w') - try: - fp.write(decodedpayload) - fp.close() - cmd = Template(config.mta.archive_html_sanitizer).safe_substitue( - filename=tmppath) - progfp = os.popen(cmd, 'r') - decodedpayload = progfp.read() - status = progfp.close() - if status: - log.error('HTML sanitizer exited with non-zero status: %s', - status) - finally: - os.unlink(tmppath) - # BAW: Since we've now sanitized the document, it should be plain - # text. Blarg, we really want the sanitizer to tell us what the type - # if the return data is. :( - ext = '.txt' - path = base + '.txt' - # Is it a message/rfc822 attachment? - elif ctype == 'message/rfc822': - submsg = msg.get_payload() - # BAW: I'm sure we can eventually do better than this. :( - decodedpayload = websafe(str(submsg)) - fp = open(path, 'w') - fp.write(decodedpayload) - fp.close() - # Now calculate the url to the list's archive. - scrubber_path = config.scrubber.archive_scrubber - base_url = find_name(scrubber_path).list_url(mlist) - if not base_url.endswith('/'): - base_url += '/' - # Trailing space will definitely be a problem with format=flowed. - # Bracket the URL instead. - url = '<' + base_url + '%s/%s%s%s>' % ( - attachments_dir, filebase, extra, ext) - return url - - - -class Scrubber: - """Cleanse a message for archiving.""" - - implements(IHandler) - - name = 'scrubber' - description = _('Cleanse a message for archiving.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/tests/test_scrubber.py b/src/mailman/pipeline/tests/test_scrubber.py deleted file mode 100644 index 7ac5eb855..000000000 --- a/src/mailman/pipeline/tests/test_scrubber.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Scrubber module tests.""" - -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type -__all__ = [ - 'TestScrubber', - ] - - -import unittest - -from mailman.pipeline import scrubber - - - -class TestScrubber(unittest.TestCase): - """Scrubber module tests.""" - - def test_guess_extension(self): - # A known extension should be found. - extension = scrubber.guess_extension('application/msword', '.doc') - self.assertEqual(extension, '.doc') - - def test_guess_missing_extension(self): - # Maybe some other extension is better. - extension = scrubber.guess_extension('application/msword', '.xxx') - self.assertEqual(extension, '.doc') diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index 8872f6ca8..e791dcfbd 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -53,8 +53,8 @@ class _AddressBase(resource.Resource, CollectionMixin): self_link=path_to('addresses/{0}'.format(address.email)), ) # Add optional attributes. These can be None or the empty string. - if address.real_name: - representation['real_name'] = address.real_name + if address.display_name: + representation['display_name'] = address.display_name if address.verified_on: representation['verified_on'] = address.verified_on return representation diff --git a/src/mailman/rest/configuration.py b/src/mailman/rest/configuration.py index 5caf53d3c..d6b27cc6c 100644 --- a/src/mailman/rest/configuration.py +++ b/src/mailman/rest/configuration.py @@ -195,7 +195,7 @@ ATTRIBUTES = dict( post_id=GetterSetter(None), posting_address=GetterSetter(None), posting_pipeline=GetterSetter(pipeline_validator), - real_name=GetterSetter(unicode), + display_name=GetterSetter(unicode), reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), request_address=GetterSetter(None), scheme=GetterSetter(None), diff --git a/src/mailman/rest/docs/addresses.rst b/src/mailman/rest/docs/addresses.rst index 71ab1f0f3..a8f875d12 100644 --- a/src/mailman/rest/docs/addresses.rst +++ b/src/mailman/rest/docs/addresses.rst @@ -77,10 +77,10 @@ the REST API. >>> cris = user_manager.create_address('cris@example.com', 'Cris Person') >>> transaction.commit() >>> dump_json('http://localhost:9001/3.0/addresses/cris@example.com') + display_name: Cris Person email: cris@example.com http_etag: "..." original_email: cris@example.com - real_name: Cris Person registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/cris@example.com @@ -115,10 +115,10 @@ addresses live in the /addresses namespace. >>> transaction.commit() >>> dump_json('http://localhost:9001/3.0/users/dave@example.com/addresses') entry 0: + display_name: Dave Person email: dave@example.com http_etag: "..." original_email: dave@example.com - real_name: Dave Person registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/dave@example.com http_etag: "..." @@ -126,10 +126,10 @@ addresses live in the /addresses namespace. total_size: 1 >>> dump_json('http://localhost:9001/3.0/addresses/dave@example.com') + display_name: Dave Person email: dave@example.com http_etag: "..." original_email: dave@example.com - real_name: Dave Person registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/dave@example.com diff --git a/src/mailman/rest/docs/configuration.rst b/src/mailman/rest/docs/configuration.rst index 1f83ff262..676b3426c 100644 --- a/src/mailman/rest/docs/configuration.rst +++ b/src/mailman/rest/docs/configuration.rst @@ -37,6 +37,7 @@ All readable attributes for a list are available on a sub-resource. description: digest_last_sent_at: None digest_size_threshold: 30.0 + display_name: Test-one filter_content: False fqdn_listname: test-one@example.com generic_nonmember_action: 1 @@ -54,7 +55,6 @@ All readable attributes for a list are available on a sub-resource. post_id: 1 posting_address: test-one@example.com posting_pipeline: default-posting-pipeline - real_name: Test-one reply_goes_to_list: no_munging request_address: test-one-request@example.com scheme: http @@ -88,7 +88,7 @@ all the writable attributes in one request. ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -136,13 +136,13 @@ These values are changed permanently. description: This is my mailing list ... digest_size_threshold: 10.5 + display_name: Fnords filter_content: True ... include_list_post_header: False include_rfc2369_headers: False ... posting_pipeline: virgin - real_name: Fnords reply_goes_to_list: point_to_list ... send_welcome_message: False @@ -168,7 +168,7 @@ must be included. It is an error to leave one or more out... ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -208,7 +208,7 @@ must be included. It is an error to leave one or more out... ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -241,7 +241,7 @@ It is also an error to spell an attribute value incorrectly... ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -273,7 +273,7 @@ It is also an error to spell an attribute value incorrectly... ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -305,7 +305,7 @@ It is also an error to spell an attribute value incorrectly... ... autoresponse_owner_text='the owner', ... autoresponse_postings_text='the mailing list', ... autoresponse_request_text='the robot', - ... real_name='Fnords', + ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, ... include_list_post_header=False, @@ -328,7 +328,7 @@ Using ``PATCH``, you can change just one attribute. >>> dump_json('http://localhost:9001/3.0/lists/' ... 'test-one@example.com/config', - ... dict(real_name='My List'), + ... dict(display_name='My List'), ... 'PATCH') content-length: 0 date: ... @@ -337,7 +337,7 @@ Using ``PATCH``, you can change just one attribute. These values are changed permanently. - >>> print mlist.real_name + >>> print mlist.display_name My List diff --git a/src/mailman/rest/docs/domains.rst b/src/mailman/rest/docs/domains.rst index a8a4fd027..c890af7fa 100644 --- a/src/mailman/rest/docs/domains.rst +++ b/src/mailman/rest/docs/domains.rst @@ -136,10 +136,13 @@ example.com domain does not contain any mailing lists. >>> dump_json('http://localhost:9001/3.0/domains/example.com/lists') entry 0: + display_name: Test-domains fqdn_listname: test-domains@example.com http_etag: "..." ... + member_count: 0 self_link: http://localhost:9001/3.0/lists/test-domains@example.com + volume: 1 http_etag: "..." start: 0 total_size: 1 diff --git a/src/mailman/rest/docs/lists.rst b/src/mailman/rest/docs/lists.rst index c412f6edc..610244968 100644 --- a/src/mailman/rest/docs/lists.rst +++ b/src/mailman/rest/docs/lists.rst @@ -20,12 +20,14 @@ Create a mailing list in a domain and it's accessible via the API. >>> dump_json('http://localhost:9001/3.0/lists') entry 0: + display_name: Test-one fqdn_listname: test-one@example.com http_etag: "..." list_name: test-one mail_host: example.com - real_name: Test-one + member_count: 0 self_link: http://localhost:9001/3.0/lists/test-one@example.com + volume: 1 http_etag: "..." start: 0 total_size: 1 @@ -35,12 +37,14 @@ You can also query for lists from a particular domain. >>> dump_json('http://localhost:9001/3.0/domains/example.com/lists') entry 0: + display_name: Test-one fqdn_listname: test-one@example.com http_etag: "..." list_name: test-one mail_host: example.com - real_name: Test-one + member_count: 0 self_link: http://localhost:9001/3.0/lists/test-one@example.com + volume: 1 http_etag: "..." start: 0 total_size: 1 @@ -82,12 +86,14 @@ The mailing list exists in the database. It is also available via the location given in the response. >>> dump_json('http://localhost:9001/3.0/lists/test-two@example.com') + display_name: Test-two fqdn_listname: test-two@example.com http_etag: "..." list_name: test-two mail_host: example.com - real_name: Test-two + member_count: 0 self_link: http://localhost:9001/3.0/lists/test-two@example.com + volume: 1 However, you are not allowed to create a mailing list in a domain that does not exist. diff --git a/src/mailman/rest/docs/membership.rst b/src/mailman/rest/docs/membership.rst index e339291aa..860d33c21 100644 --- a/src/mailman/rest/docs/membership.rst +++ b/src/mailman/rest/docs/membership.rst @@ -480,7 +480,7 @@ get gets a regular delivery. >>> dump_json('http://localhost:9001/3.0/members', { ... 'fqdn_listname': 'ant@example.com', ... 'subscriber': 'eperson@example.com', - ... 'real_name': 'Elly Person', + ... 'display_name': 'Elly Person', ... }) content-length: 0 date: ... @@ -608,7 +608,7 @@ Fred joins the `ant` mailing list but wants MIME digest delivery. >>> dump_json('http://localhost:9001/3.0/members', { ... 'fqdn_listname': 'ant@example.com', ... 'subscriber': 'fperson@example.com', - ... 'real_name': 'Fred Person', + ... 'display_name': 'Fred Person', ... 'delivery_mode': 'mime_digests', ... }) content-length: 0 diff --git a/src/mailman/rest/docs/users.rst b/src/mailman/rest/docs/users.rst index 145b069d9..a20306e17 100644 --- a/src/mailman/rest/docs/users.rst +++ b/src/mailman/rest/docs/users.rst @@ -26,8 +26,8 @@ When there are users in the database, they can be retrieved as a collection. >>> dump_json('http://localhost:9001/3.0/users') entry 0: created_on: 2005-08-01T07:49:23 + display_name: Anne Person http_etag: "..." - real_name: Anne Person self_link: http://localhost:9001/3.0/users/1 user_id: 1 http_etag: "..." @@ -48,8 +48,8 @@ returned in the REST API. >>> dump_json('http://localhost:9001/3.0/users') entry 0: created_on: 2005-08-01T07:49:23 + display_name: Anne Person http_etag: "..." - real_name: Anne Person self_link: http://localhost:9001/3.0/users/1 user_id: 1 entry 1: @@ -72,7 +72,7 @@ email address for the user, a password, and optionally the user's full name. >>> transaction.abort() >>> dump_json('http://localhost:9001/3.0/users', { ... 'email': 'bart@example.com', - ... 'real_name': 'Bart Person', + ... 'display_name': 'Bart Person', ... 'password': 'bbb', ... }) content-length: 0 @@ -92,9 +92,9 @@ It is also available via the location given in the response. >>> dump_json('http://localhost:9001/3.0/users/3') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -103,9 +103,9 @@ them with user ids. Thus, a user can be retrieved via its email address. >>> dump_json('http://localhost:9001/3.0/users/bart@example.com') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -117,7 +117,7 @@ therefore cannot be retrieved. It can be reset though. >>> transaction.abort() >>> dump_json('http://localhost:9001/3.0/users', { ... 'email': 'cris@example.com', - ... 'real_name': 'Cris Person', + ... 'display_name': 'Cris Person', ... }) content-length: 0 date: ... @@ -127,9 +127,9 @@ therefore cannot be retrieved. It can be reset though. >>> dump_json('http://localhost:9001/3.0/users/4') created_on: 2005-08-01T07:49:23 + display_name: Cris Person http_etag: "..." password: {CLEARTEXT}... - real_name: Cris Person self_link: http://localhost:9001/3.0/users/4 user_id: 4 @@ -204,10 +204,10 @@ sorted in lexical order by original (i.e. case-preserved) email address. registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/bart.person@example.com entry 2: + display_name: Bart Person email: bart@example.com http_etag: "..." original_email: bart@example.com - real_name: Bart Person registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/bart@example.com entry 3: @@ -225,32 +225,32 @@ In fact, any of these addresses can be used to look up Bart's user record. >>> dump_json('http://localhost:9001/3.0/users/bart@example.com') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 >>> dump_json('http://localhost:9001/3.0/users/bart.person@example.com') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 >>> dump_json('http://localhost:9001/3.0/users/bperson@example.com') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 >>> dump_json('http://localhost:9001/3.0/users/Bart.Q.Person@example.com') created_on: 2005-08-01T07:49:23 + display_name: Bart Person http_etag: "..." password: {CLEARTEXT}bbb - real_name: Bart Person self_link: http://localhost:9001/3.0/users/3 user_id: 3 diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py index 2ce62b0fe..2824a894e 100644 --- a/src/mailman/rest/helpers.py +++ b/src/mailman/rest/helpers.py @@ -77,10 +77,10 @@ class ExtendedEncoder(json.JSONEncoder): seconds = obj.seconds + obj.microseconds / 1000000.0 return '{0}d{1}s'.format(obj.days, seconds) return '{0}d'.format(obj.days) - elif hasattr(obj, 'enumclass') and issubclass(obj.enumclass, Enum): + elif hasattr(obj, 'enum') and issubclass(obj.enum, Enum): # It's up to the decoding validator to associate this name with # the right Enum class. - return obj.enumname + return obj.name return json.JSONEncoder.default(self, obj) diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 0103022e7..3374e8f73 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -105,10 +105,12 @@ class _ListBase(resource.Resource, CollectionMixin): def _resource_as_dict(self, mlist): """See `CollectionMixin`.""" return dict( + display_name=mlist.display_name, fqdn_listname=mlist.fqdn_listname, - mail_host=mlist.mail_host, list_name=mlist.list_name, - real_name=mlist.real_name, + mail_host=mlist.mail_host, + member_count=mlist.members.member_count, + volume=mlist.volume, self_link=path_to('lists/{0}'.format(mlist.fqdn_listname)), ) @@ -135,10 +137,7 @@ class AList(_ListBase): """Delete the named mailing list.""" if self._mlist is None: return http.not_found() - remove_list(self._mlist.fqdn_listname, self._mlist, - # XXX 2010-07-06 barry we need a way to remove the list - # archives either with the mailing list or afterward. - archives=False) + remove_list(self._mlist.fqdn_listname, self._mlist) return no_content() @resource.child(member_matcher) diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index d55aa7dc2..761e3147c 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -199,10 +199,10 @@ class AllMembers(_MemberBase): validator = Validator( fqdn_listname=unicode, subscriber=subscriber_validator, - real_name=unicode, + display_name=unicode, delivery_mode=enum_validator(DeliveryMode), role=enum_validator(MemberRole), - _optional=('delivery_mode', 'real_name', 'role')) + _optional=('delivery_mode', 'display_name', 'role')) member = service.join(**validator(request)) except AlreadySubscribedError: return http.conflict([], b'Member already subscribed') diff --git a/src/mailman/rest/tests/test_lists.py b/src/mailman/rest/tests/test_lists.py index 78b793c6c..b030a2e8d 100644 --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -17,23 +17,31 @@ """REST list tests.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestLists', + 'TestListsMissing', ] import unittest from urllib2 import HTTPError +from zope.component import getUtility +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer -class TestLists(unittest.TestCase): +class TestListsMissing(unittest.TestCase): + """Test expected failures.""" + layer = RESTLayer def test_missing_list_roster_member_404(self): @@ -79,3 +87,44 @@ class TestLists(unittest.TestCase): self.assertEqual(exc.code, 404) else: raise AssertionError('Expected HTTPError') + + + +class TestLists(unittest.TestCase): + """Test various aspects of mailing list resources.""" + + layer = RESTLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + config.db.commit() + self._usermanager = getUtility(IUserManager) + + def test_member_count_with_no_members(self): + # The list initially has 0 members. + resource, response = call_api( + 'http://localhost:9001/3.0/lists/test@example.com') + self.assertEqual(response.status, 200) + self.assertEqual(resource['member_count'], 0) + + def test_member_count_with_one_member(self): + # Add a member to a list and check that the resource reflects this. + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) + config.db.commit() + resource, response = call_api( + 'http://localhost:9001/3.0/lists/test@example.com') + self.assertEqual(response.status, 200) + self.assertEqual(resource['member_count'], 1) + + def test_member_count_with_two_members(self): + # Add two members to a list and check that the resource reflects this. + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) + bart = self._usermanager.create_address('bar@example.com') + self._mlist.subscribe(bart) + config.db.commit() + resource, response = call_api( + 'http://localhost:9001/3.0/lists/test@example.com') + self.assertEqual(response.status, 200) + self.assertEqual(resource['member_count'], 2) diff --git a/src/mailman/rest/tests/test_membership.py b/src/mailman/rest/tests/test_membership.py index 568a56306..202e5f057 100644 --- a/src/mailman/rest/tests/test_membership.py +++ b/src/mailman/rest/tests/test_membership.py @@ -124,7 +124,7 @@ class TestMembership(unittest.TestCase): call_api('http://localhost:9001/3.0/members', { 'fqdn_listname': 'test@example.com', 'subscriber': 'anne@example.com', - 'real_name': 'Anne Person', + 'display_name': 'Anne Person', 'delivery_mode': 'invalid-mode', }) except HTTPError as exc: @@ -138,7 +138,7 @@ class TestMembership(unittest.TestCase): content, response = call_api('http://localhost:9001/3.0/members', { 'fqdn_listname': 'test@example.com', 'subscriber': 'hugh/person@example.com', - 'real_name': 'Hugh Person', + 'display_name': 'Hugh Person', }) self.assertEqual(content, None) self.assertEqual(response.status, 201) diff --git a/src/mailman/rest/users.py b/src/mailman/rest/users.py index 0a1503930..4e1362120 100644 --- a/src/mailman/rest/users.py +++ b/src/mailman/rest/users.py @@ -60,8 +60,8 @@ class _UserBase(resource.Resource, CollectionMixin): # with the real name. These could be None or the empty string. if user.password: resource['password'] = user.password - if user.real_name: - resource['real_name'] = user.real_name + if user.display_name: + resource['display_name'] = user.display_name return resource def _get_collection(self, request): @@ -84,9 +84,9 @@ class AllUsers(_UserBase): """Create a new user.""" try: validator = Validator(email=unicode, - real_name=unicode, + display_name=unicode, password=unicode, - _optional=('real_name', 'password')) + _optional=('display_name', 'password')) arguments = validator(request) except ValueError as error: return http.bad_request([], str(error)) diff --git a/src/mailman/rules/administrivia.py b/src/mailman/rules/administrivia.py index 790c16c19..41c6edf30 100644 --- a/src/mailman/rules/administrivia.py +++ b/src/mailman/rules/administrivia.py @@ -81,7 +81,7 @@ class Administrivia: lineno = 0 for line in lines: line = line.strip() - if line == '': + if len(line) == 0: continue lineno += 1 if lineno > config.mailman.email_commands_max_lines: diff --git a/src/mailman/rules/moderation.py b/src/mailman/rules/moderation.py index bcec47cba..cb27d89d8 100644 --- a/src/mailman/rules/moderation.py +++ b/src/mailman/rules/moderation.py @@ -57,7 +57,7 @@ class MemberModeration: elif action is not None: # We must stringify the moderation action so that it can be # stored in the pending request table. - msgdata['moderation_action'] = action.enumname + msgdata['moderation_action'] = action.name msgdata['moderation_sender'] = sender return True # The sender is not a member so this rule does not match. @@ -98,7 +98,7 @@ class NonmemberModeration: elif action is not None: # We must stringify the moderation action so that it can be # stored in the pending request table. - msgdata['moderation_action'] = action.enumname + msgdata['moderation_action'] = action.name msgdata['moderation_sender'] = sender return True # The sender must be a member, so this rule does not match. diff --git a/src/mailman/rules/suspicious.py b/src/mailman/rules/suspicious.py index a06a2d0aa..ad1ab42cd 100644 --- a/src/mailman/rules/suspicious.py +++ b/src/mailman/rules/suspicious.py @@ -68,7 +68,7 @@ def _parse_matching_header_opt(mlist): # This didn't look like a header line. BAW: should do a # better job of informing the list admin. log.error('bad bounce_matching_header line: %s\n%s', - mlist.real_name, line) + mlist.display_name, line) else: header = line[:i] value = line[i+1:].lstrip() @@ -79,7 +79,7 @@ def _parse_matching_header_opt(mlist): # job of informing the list admin. log.error("""\ bad regexp in bounce_matching_header line: %s -\n%s (cause: %s)""", mlist.real_name, value, error) +\n%s (cause: %s)""", mlist.display_name, value, error) else: all.append((header, cre, line)) return all diff --git a/src/mailman/runners/archive.py b/src/mailman/runners/archive.py index cab776076..7295a5c57 100644 --- a/src/mailman/runners/archive.py +++ b/src/mailman/runners/archive.py @@ -17,73 +17,91 @@ """Archive runner.""" +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type __all__ = [ 'ArchiveRunner', ] -import os +import copy import logging +from email.utils import parsedate_tz, mktime_tz from datetime import datetime -from email.utils import parsedate_tz, mktime_tz, formatdate -from flufl.lock import Lock from lazr.config import as_timedelta from mailman.config import config from mailman.core.runner import Runner +from mailman.interfaces.archiver import ClobberDate +from mailman.utilities.datetime import RFC822_DATE_FMT, now + log = logging.getLogger('mailman.error') +def _should_clobber(msg, msgdata, archiver): + """Should the Date header in the original message get clobbered?""" + # Calculate the Date header of the message as a datetime. What if there + # are multiple Date headers, even in violation of the RFC? For now, take + # the first one. If there are no Date headers, then definitely clobber. + original_date = msg.get('date') + if original_date is None: + return True + section = getattr(config.archiver, archiver, None) + if section is None: + log.error('No archiver config section found: {0}'.format(archiver)) + return False + try: + clobber = ClobberDate[section.clobber_date] + except ValueError: + log.error('Invalid clobber_date for "{0}": {1}'.format( + archiver, section.clobber_date)) + return False + if clobber is ClobberDate.always: + return True + elif clobber is ClobberDate.never: + return False + # Maybe we'll clobber the date. Let's see if it's farther off from now + # than the skew period. + skew = as_timedelta(section.clobber_skew) + try: + time_tuple = parsedate_tz(original_date) + except (ValueError, OverflowError): + # The likely cause of this is that the year in the Date: field is + # horribly incorrect, e.g. (from SF bug # 571634): + # + # Date: Tue, 18 Jun 0102 05:12:09 +0500 + # + # Obviously clobber such dates. + return True + if time_tuple is None: + # There was some other bogosity in the Date header. + return True + claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple)) + return (abs(now() - claimed_date) > skew) + + + class ArchiveRunner(Runner): """The archive runner.""" def _dispose(self, mlist, msg, msgdata): - # Support clobber_date, i.e. setting the date in the archive to the - # received date, not the (potentially bogus) Date: header of the - # original message. - clobber = False - original_date = msg.get('date') - received_time = formatdate(msgdata['received_time']) - if not original_date: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 1: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 2: - # What's the timestamp on the original message? - timetup = parsedate_tz(original_date) - now = datetime.now() + received_time = msgdata.get('received_time', now(strip_tzinfo=False)) + for archiver in config.archivers: + msg_copy = copy.deepcopy(msg) + if _should_clobber(msg, msgdata, archiver.name): + original_date = msg_copy['date'] + del msg_copy['date'] + del msg_copy['x-original-date'] + msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT) + if original_date: + msg_copy['X-Original-Date'] = original_date + # A problem in one archiver should not prevent other archivers + # from running. try: - if not timetup: - clobber = True - else: - utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) - date_skew = as_timedelta( - config.archiver.pipermail.allowable_sane_date_skew) - clobber = (abs(now - utc_timestamp) > date_skew) - except (ValueError, OverflowError): - # The likely cause of this is that the year in the Date: field - # is horribly incorrect, e.g. (from SF bug # 571634): - # Date: Tue, 18 Jun 0102 05:12:09 +0500 - # Obviously clobber such dates. - clobber = True - if clobber: - del msg['date'] - del msg['x-original-date'] - msg['Date'] = received_time - if original_date: - msg['X-Original-Date'] = original_date - # Always put an indication of when we received the message. - msg['X-List-Received-Date'] = received_time - # While a list archiving lock is acquired, archive the message. - with Lock(os.path.join(mlist.data_path, 'archive.lck')): - for archiver in config.archivers: - # A problem in one archiver should not prevent other archivers - # from running. - try: - archiver.archive_message(mlist, msg) - except Exception: - log.exception('Broken archiver: %s' % archiver.name) + archiver.archive_message(mlist, msg_copy) + except Exception: + log.exception('Broken archiver: %s' % archiver.name) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 2730fc427..afc11f732 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -41,12 +41,10 @@ from email.utils import formatdate, getaddresses, make_msgid from urllib2 import URLError from mailman.config import config -from mailman.core.errors import DiscardMessage from mailman.core.i18n import _ from mailman.core.runner import Runner +from mailman.handlers.decorate import decorate from mailman.interfaces.member import DeliveryMode, DeliveryStatus -from mailman.pipeline.decorate import decorate -from mailman.pipeline.scrubber import process as scrubber from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox from mailman.utilities.string import oneline, wrap @@ -63,9 +61,8 @@ class Digester: self._mlist = mlist self._charset = mlist.preferred_language.charset # This will be used in the Subject, so use $-strings. - realname = mlist.real_name - issue = digest_number - self._digest_id = _('$realname Digest, Vol $volume, Issue $issue') + self._digest_id = _( + '$mlist.display_name Digest, Vol $volume, Issue $digest_number') self._subject = Header(self._digest_id, self._charset, header_name='Subject') @@ -83,7 +80,7 @@ class Digester: # ahead and add it now. self._masthead = make('masthead.txt', mailing_list=mlist, - real_name=mlist.real_name, + display_name=mlist.display_name, got_list_email=mlist.posting_address, got_listinfo_url=mlist.script_url('listinfo'), got_request_email=mlist.request_address, @@ -253,12 +250,6 @@ class RFC1153Digester(Digester): if count > 1: print >> self._text, self._separator30 print >> self._text - # Scrub attachements. - try: - msg = scrubber(self._mlist, msg) - except DiscardMessage: - print >> self._text, _('[Message discarded by content filter]') - return # Each message section contains a few headers. for header in config.digests.plain_digest_keep_headers.split(): if header in msg: diff --git a/src/mailman/runners/docs/archiver.rst b/src/mailman/runners/docs/archiver.rst deleted file mode 100644 index a6f5ccd24..000000000 --- a/src/mailman/runners/docs/archiver.rst +++ /dev/null @@ -1,35 +0,0 @@ -========= -Archiving -========= - -Mailman can archive to any number of archivers that adhere to the -``IArchiver`` interface. By default, there's a Pipermail archiver. -:: - - >>> mlist = create_list('test@example.com') - >>> transaction.commit() - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: test@example.com - ... Subject: My first post - ... Message-ID: <first> - ... - ... First post! - ... """) - - >>> archiver_queue = config.switchboards['archive'] - >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname) - - >>> from mailman.runners.archive import ArchiveRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> runner = make_testable_runner(ArchiveRunner) - >>> runner.run() - - # The best we can do is verify some landmark exists. Let's use the - # Pipermail pickle file exists. - >>> listname = mlist.fqdn_listname - >>> import os - >>> os.path.exists(os.path.join( - ... config.PUBLIC_ARCHIVE_FILE_DIR, listname, 'pipermail.pck')) - True diff --git a/src/mailman/runners/docs/digester.rst b/src/mailman/runners/docs/digester.rst index 5a20db556..1ca51bdae 100644 --- a/src/mailman/runners/docs/digester.rst +++ b/src/mailman/runners/docs/digester.rst @@ -223,7 +223,6 @@ The RFC 1153 contains the digest in a single plain text message. When replying, please edit your Subject line so it is more specific than "Re: Contents of Test digest..." <BLANKLINE> - <BLANKLINE> Today's Topics: <BLANKLINE> 1. Test message 1 (aperson@example.com) @@ -237,7 +236,6 @@ The RFC 1153 contains the digest in a single plain text message. From: aperson@example.com Subject: Test message 1 To: xtest@example.com - Message-ID: ... <BLANKLINE> Here is message 1 <BLANKLINE> @@ -246,7 +244,6 @@ The RFC 1153 contains the digest in a single plain text message. From: aperson@example.com Subject: Test message 2 To: xtest@example.com - Message-ID: ... <BLANKLINE> Here is message 2 <BLANKLINE> @@ -255,7 +252,6 @@ The RFC 1153 contains the digest in a single plain text message. From: aperson@example.com Subject: Test message 3 To: xtest@example.com - Message-ID: ... <BLANKLINE> Here is message 3 <BLANKLINE> @@ -264,7 +260,6 @@ The RFC 1153 contains the digest in a single plain text message. From: aperson@example.com Subject: Test message 4 To: xtest@example.com - Message-ID: ... <BLANKLINE> Here is message 4 <BLANKLINE> @@ -346,7 +341,7 @@ You can see that the digests contain a mix of French and Japanese. Content-Type: multipart/mixed; boundary="===============...==" MIME-Version: 1.0 From: test-request@example.com - Subject: Groupe Test, Vol. 1, Parution 2 + Subject: Groupe Test, Vol 1, Parution 2 To: test@example.com Reply-To: test@example.com Date: ... @@ -356,7 +351,7 @@ You can see that the digests contain a mix of French and Japanese. Content-Type: text/plain; charset="iso-8859-1" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable - Content-Description: Groupe Test, Vol. 1, Parution 2 + Content-Description: Groupe Test, Vol 1, Parution 2 <BLANKLINE> Envoyez vos messages pour la liste Test =E0 test@example.com @@ -413,7 +408,7 @@ French and Japanese characters. >>> print rfc1153.msg.as_string() From: test-request@example.com - Subject: Groupe Test, Vol. 1, Parution 2 + Subject: Groupe Test, Vol 1, Parution 2 To: test@example.com Reply-To: test@example.com Date: ... @@ -458,7 +453,6 @@ The content can be decoded to see the actual digest text. "'From: aperson@example.org'", "'Subject: \\xe4\\xb8\\x80\\xe7\\x95\\xaa'", "'To: test@example.com'", - "'Message-ID: ... "'Content-Type: text/plain; charset=iso-2022-jp'", "''", "'\\xe4\\xb8\\x80\\xe7\\x95\\xaa'", @@ -471,8 +465,8 @@ The content can be decoded to see the actual digest text. "'http://lists.example.com/listinfo/test@example.com'", "''", "''", - "'Fin de Groupe Test, Vol. 1, Parution 2'", - "'**************************************'"] + "'Fin de Groupe Test, Vol 1, Parution 2'", + "'*************************************'"] >>> config.pop('french') diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst index 2b6c4b42b..3ce145907 100644 --- a/src/mailman/runners/docs/lmtp.rst +++ b/src/mailman/runners/docs/lmtp.rst @@ -306,7 +306,7 @@ Messages to the `-owner` address also go to the incoming processor. 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : mylist@example.com original_size: ... subaddress : owner diff --git a/src/mailman/runners/docs/outgoing.rst b/src/mailman/runners/docs/outgoing.rst index b1ffb06a0..d8963ae00 100644 --- a/src/mailman/runners/docs/outgoing.rst +++ b/src/mailman/runners/docs/outgoing.rst @@ -33,7 +33,7 @@ move messages to the 'retry queue' for handling delivery failures. Normally, messages would show up in the outgoing queue after the message has been processed by the rule set and pipeline. But we can simulate that here by injecting a message directly into the outgoing queue. First though, we must -call the ``calculate-recipients`` handler so that the message metadata will be +call the ``member-recipients`` handler so that the message metadata will be populated with the list of addresses to deliver the message to. :: @@ -47,7 +47,7 @@ populated with the list of addresses to deliver the message to. ... """) >>> msgdata = {} - >>> handler = config.handlers['calculate-recipients'] + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> outgoing_queue = config.switchboards['out'] diff --git a/src/mailman/runners/incoming.py b/src/mailman/runners/incoming.py index 7072f9bcc..d8db926c7 100644 --- a/src/mailman/runners/incoming.py +++ b/src/mailman/runners/incoming.py @@ -61,6 +61,9 @@ class IncomingRunner(Runner): pass config.db.commit() # Process the message through the mailing list's start chain. - process(mlist, msg, msgdata, mlist.posting_chain) + start_chain = (mlist.owner_chain + if msgdata.get('to_owner', False) + else mlist.posting_chain) + process(mlist, msg, msgdata, start_chain) # Do not keep this message queued. return False diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index bee111ad1..45fa5a783 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -44,8 +44,10 @@ from mailman.core.runner import Runner from mailman.database.transaction import txn from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager +from mailman.utilities.datetime import now from mailman.utilities.email import add_message_hash + elog = logging.getLogger('mailman.error') qlog = logging.getLogger('mailman.runner') slog = logging.getLogger('mailman.smtp') @@ -181,6 +183,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # see if it's destined for a valid mailing list. If so, then queue # the message to the appropriate place and record a 250 status for # that recipient. If not, record a failure status for that recipient. + received_time = now() for to in rcpttos: try: to = parseaddr(to)[1].lower() @@ -196,7 +199,8 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # queue. queue = None msgdata = dict(listname=listname, - original_size=msg.original_size) + original_size=msg.original_size, + received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) queue = SUBADDRESS_QUEUES.get(canonical_subaddress) if subaddress is None: diff --git a/src/mailman/runners/pipeline.py b/src/mailman/runners/pipeline.py index 8bee2c4cb..b031662b3 100644 --- a/src/mailman/runners/pipeline.py +++ b/src/mailman/runners/pipeline.py @@ -30,6 +30,9 @@ from mailman.core.runner import Runner class PipelineRunner(Runner): def _dispose(self, mlist, msg, msgdata): # Process the message through the mailing list's pipeline. - process(mlist, msg, msgdata, mlist.posting_pipeline) + pipeline = (mlist.owner_pipeline + if msgdata.get('to_owner', False) + else mlist.posting_pipeline) + process(mlist, msg, msgdata, pipeline) # Do not keep this message queued. return False diff --git a/src/mailman/runners/tests/test_archiver.py b/src/mailman/runners/tests/test_archiver.py new file mode 100644 index 000000000..ca09de9fa --- /dev/null +++ b/src/mailman/runners/tests/test_archiver.py @@ -0,0 +1,260 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the archive runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestArchiveRunner', + ] + + +import os +import unittest + +from email import message_from_file +from zope.interface import implements + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.archiver import IArchiver +from mailman.runners.archive import ArchiveRunner +from mailman.testing.helpers import ( + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import RFC822_DATE_FMT, factory, now + + + +# This helper will set up a specific archiver as appropriate for a specific +# test. It assumes the setUp() will just disable all archivers. +def archiver(name, enable=False, clobber=None, skew=None): + def decorator(func): + def wrapper(*args, **kws): + config_name = 'archiver {0}'.format(name) + section = """ + [archiver.{0}] + enable: {1} + clobber_date: {2} + clobber_skew: {3} + """.format(name, + 'yes' if enable else 'no', + clobber, skew) + config.push(config_name, section) + try: + return func(*args, **kws) + finally: + config.pop(config_name) + return wrapper + return decorator + + + +class DummyArchiver: + implements(IArchiver) + name = 'dummy' + + @staticmethod + def list_url(mlist): + return 'http://archive.example.com/' + + @staticmethod + def permalink(mlist, msg): + filename = msg['x-message-id-hash'] + return 'http://archive.example.com/' + filename + + @staticmethod + def archive_message(mlist, msg): + filename = msg['x-message-id-hash'] + path = os.path.join(config.MESSAGES_DIR, filename) + with open(path, 'w') as fp: + print(msg.as_string(), file=fp) + # Not technically allowed by the API, but good enough for the test. + return path + + + +class TestArchiveRunner(unittest.TestCase): + """Test the archive runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._now = now() + # Enable just the dummy archiver. + config.push('dummy', """ + [archiver.dummy] + class: mailman.runners.tests.test_archiver.DummyArchiver + enable: no + [archiver.prototype] + enable: no + [archiver.mhonarc] + enable: no + [archiver.mail_archive] + enable: no + """) + self._archiveq = config.switchboards['archive'] + self._msg = mfs("""\ +From: aperson@example.com +To: test@example.com +Subject: My first post +Message-ID: <first> +X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB + +First post! +""") + self._runner = make_testable_runner(ArchiveRunner) + + def tearDown(self): + config.pop('dummy') + + @archiver('dummy', enable=True) + def test_archive_runner(self): + # Ensure that the archive runner ends up archiving the message. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + + @archiver('dummy', enable=True) + def test_archive_runner_with_dated_message(self): + # Date headers don't throw off the archiver runner. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='never') + def test_clobber_date_never(self): + # Even if the Date header is insanely off from the received time of + # the message, if clobber_date is 'never', the header is not clobbered. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True) + def test_clobber_dateless(self): + # A message with no Date header will always get clobbered. + self.assertEqual(self._msg['date'], None) + # Now, before enqueuing the message (well, really, calling 'now()' + # again), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now(strip_tzinfo=False)) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='always') + def test_clobber_date_always(self): + # The date always gets clobbered with the current received time. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='maybe', skew='1d') + def test_clobber_date_maybe_when_insane(self): + # The date is clobbered if it's farther off from now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='maybe', skew='10d') + def test_clobber_date_maybe_when_sane(self): + # The date is not clobbered if it's nearer to now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], None) diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py index ea972c17f..d2b24a2d1 100644 --- a/src/mailman/runners/tests/test_confirm.py +++ b/src/mailman/runners/tests/test_confirm.py @@ -21,13 +21,13 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestConfirm', ] import unittest from datetime import datetime -from email.iterators import body_line_iterator from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -36,6 +36,7 @@ from mailman.interfaces.registrar import IRegistrar from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner from mailman.testing.helpers import ( + body_line_iterator, get_queue_messages, make_testable_runner, specialized_message_from_string as mfs) diff --git a/src/mailman/runners/tests/test_incoming.py b/src/mailman/runners/tests/test_incoming.py new file mode 100644 index 000000000..5a0d82765 --- /dev/null +++ b/src/mailman/runners/tests/test_incoming.py @@ -0,0 +1,94 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the incoming queue runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestIncoming', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.base import TerminalChainBase +from mailman.config import config +from mailman.runners.incoming import IncomingRunner +from mailman.testing.helpers import ( + get_queue_messages, + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class Chain(TerminalChainBase): + name = 'test' + description = 'a test chain' + + def __init__(self, marker): + self._marker = marker + + def _process(self, mlist, msg, msgdata): + msgdata['marker'] = self._marker + config.switchboards['out'].enqueue(msg, msgdata) + + + +class TestIncoming(unittest.TestCase): + """Test the incoming queue runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_chain = 'test posting' + self._mlist.owner_chain = 'test owner' + config.chains['test posting'] = Chain('posting') + config.chains['test owner'] = Chain('owner') + self._in = make_testable_runner(IncomingRunner, 'in') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.chains['test posting'] + del config.chains['test owner'] + + def test_posting(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com') + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'posting') + + def test_owner(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'owner') diff --git a/src/mailman/runners/tests/test_join.py b/src/mailman/runners/tests/test_join.py index 8cbd8659f..a584fd2c2 100644 --- a/src/mailman/runners/tests/test_join.py +++ b/src/mailman/runners/tests/test_join.py @@ -21,12 +21,13 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestJoin', + 'TestJoinWithDigests', ] import unittest -from email.iterators import body_line_iterator from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -37,6 +38,7 @@ from mailman.interfaces.subscriptions import ISubscriptionService from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner from mailman.testing.helpers import ( + body_line_iterator, get_queue_messages, make_testable_runner, reset_the_world, diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 2c4defe59..87b69c7e4 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -28,6 +28,8 @@ __all__ = [ import smtplib import unittest +from datetime import datetime + from mailman.app.lifecycle import create_list from mailman.config import config from mailman.testing.helpers import get_lmtp_client, get_queue_messages @@ -96,3 +98,17 @@ Subject: This has a Message-ID but no X-Message-ID-Hash self.assertEqual(len(all_headers), 1) self.assertEqual(messages[0].msg['x-message-id-hash'], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + + def test_received_time(self): + # The LMTP runner adds a `received_time` key to the metadata. + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Subject: This has no Message-ID header +Message-ID: <ant> + +""") + messages = get_queue_messages('in') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['received_time'], + datetime(2005, 8, 1, 7, 49, 23)) diff --git a/src/mailman/runners/tests/test_owner.py b/src/mailman/runners/tests/test_owner.py new file mode 100644 index 000000000..622bb2255 --- /dev/null +++ b/src/mailman/runners/tests/test_owner.py @@ -0,0 +1,142 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test posting to a mailing list's -owner address.""" + +# XXX 2012-03-23 BAW: This is not necessarily the best place for this test. +# We really need a better place to collect these sort of end-to-end posting +# tests. They're not exactly integration tests, but they do touch lots of +# parts of the system. + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestEmailToOwner', + ] + + +import unittest + +from operator import itemgetter +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import ( + TestableMaster, + get_lmtp_client, + make_testable_runner) +from mailman.runners.incoming import IncomingRunner +from mailman.runners.outgoing import OutgoingRunner +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.layers import SMTPLayer + + + +class TestEmailToOwner(unittest.TestCase): + """Test emailing a mailing list's -owner address.""" + + layer = SMTPLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + # Add some owners, moderators, and members + manager = getUtility(IUserManager) + anne = manager.create_address('anne@example.com') + bart = manager.create_address('bart@example.com') + cris = manager.create_address('cris@example.com') + dave = manager.create_address('dave@example.com') + self._mlist.subscribe(anne, MemberRole.member) + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.owner) + self._mlist.subscribe(cris, MemberRole.moderator) + self._mlist.subscribe(dave, MemberRole.member) + config.db.commit() + self._inq = make_testable_runner(IncomingRunner, 'in') + self._pipelineq = make_testable_runner(PipelineRunner, 'pipeline') + self._outq = make_testable_runner(OutgoingRunner, 'out') + # Python 2.7 has assertMultiLineEqual. Let this work without bounds. + self.maxDiff = None + self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + + def test_owners_get_email(self): + # XXX 2012-03-23 BAW: We can't use a layer here because we need both + # the SMTPLayer and LMTPLayer and these are incompatible. There's no + # way to make zope.test* happy without causing errors or worse. Live + # with this hack until we can rip all that layer crap out and use + # something like testresources. + def wait(): + get_lmtp_client(quiet=True) + lmtpd = TestableMaster(wait) + lmtpd.start('lmtp') + # Post a message to the list's -owner address, and all the owners will + # get a copy of the message. + lmtp = get_lmtp_client(quiet=True) + lmtp.lhlo('remote.example.org') + lmtp.sendmail('zuzu@example.org', ['test-owner@example.com'], """\ +From: Zuzu Person <zuzu@example.org> +To: test-owner@example.com +Message-ID: <ant> + +Can you help me? +""") + lmtpd.stop() + # There should now be one message sitting in the incoming queue. + # Check that, then process it. Don't use get_queue_messages() since + # that will empty the queue. + self.assertEqual(len(config.switchboards['in'].files), 1) + self._inq.run() + # There should now be one message sitting in the pipeline queue. + # Process that one too. + self.assertEqual(len(config.switchboards['pipeline'].files), 1) + self._pipelineq.run() + # The message has made its way to the outgoing queue. Again, check + # and process that one. + self.assertEqual(len(config.switchboards['out'].files), 1) + self._outq.run() + # The SMTP server has now received three messages, one for each of the + # owners and moderators. Of course, Bart is both an owner and a + # moderator, so he'll get only one copy of the message. Dave does not + # get a copy of the message. + messages = sorted(SMTPLayer.smtpd.messages, key=itemgetter('x-rcptto')) + self.assertEqual(len(messages), 3) + self.assertEqual(messages[0]['x-rcptto'], 'anne@example.com') + self.assertEqual(messages[1]['x-rcptto'], 'bart@example.com') + self.assertEqual(messages[2]['x-rcptto'], 'cris@example.com') + # And yet, all three messages are addressed to the -owner address. + for message in messages: + self.assertEqual(message['to'], 'test-owner@example.com') + # All three messages will have two X-MailFrom headers. One is added + # by the LMTP server accepting Zuzu's original message, and will + # contain her posting address, i.e. zuzu@example.com. The second one + # is added by the lazr.smtptest server that accepts Mailman's VERP'd + # message to the individual recipient. By verifying both, we prove + # that Zuzu sent the original message, and that Mailman is VERP'ing + # the copy to all the owners. + self.assertEqual( + messages[0].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+anne=example.com@example.com']) + self.assertEqual( + messages[1].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+bart=example.com@example.com']) + self.assertEqual( + messages[2].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+cris=example.com@example.com']) diff --git a/src/mailman/runners/tests/test_pipeline.py b/src/mailman/runners/tests/test_pipeline.py new file mode 100644 index 000000000..8776bf844 --- /dev/null +++ b/src/mailman/runners/tests/test_pipeline.py @@ -0,0 +1,115 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the pipeline runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPipelineRunner', + ] + + +import unittest + +from zope.interface import implements + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.handler import IHandler +from mailman.interfaces.pipeline import IPipeline +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.helpers import ( + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class MyTestHandler: + implements(IHandler) + name = 'test handler' + description = 'A test handler' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def process(self, mlist, msg, msgdata): + self._test.mark(self._marker) + + +class MyTestPipeline: + implements(IPipeline) + name = 'test' + description = 'a test pipeline' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def __iter__(self): + yield MyTestHandler(self._marker, self._test) + + + +class TestPipelineRunner(unittest.TestCase): + """Test the pipeline runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_pipeline = 'test posting' + self._mlist.owner_pipeline = 'test owner' + config.pipelines['test posting'] = MyTestPipeline('posting', self) + config.pipelines['test owner'] = MyTestPipeline('owner', self) + self._pipeline = make_testable_runner(PipelineRunner, 'pipeline') + self._markers = [] + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.pipelines['test posting'] + del config.pipelines['test owner'] + + def mark(self, marker): + # Record a marker seen by a handler. + self._markers.append(marker) + + def test_posting(self): + # A message accepted for posting gets processed through the posting + # pipeline. + msgdata = dict(listname='test@example.com') + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'posting') + + def test_owner(self): + # A message accepted for posting to a list's owners gets processed + # through the owner pipeline. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'owner') diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index 95672c62c..b6900dca6 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -27,12 +27,11 @@ __all__ = [ # XXX Styles need to be reconciled with lazr.config. -import datetime - +from datetime import timedelta from zope.interface import implements from mailman.core.i18n import _ -from mailman.interfaces.action import Action +from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.autorespond import ResponseAction @@ -55,7 +54,7 @@ class DefaultStyle: # For cut-n-paste convenience. mlist = mailing_list # List identity. - mlist.real_name = mlist.list_name.capitalize() + mlist.display_name = mlist.list_name.capitalize() mlist.list_id = '{0.list_name}.{0.mail_host}'.format(mlist) mlist.include_rfc2369_headers = True mlist.include_list_post_header = True @@ -99,6 +98,7 @@ from: .*@uplinkpro.com mlist.preferred_language = 'en' mlist.collapse_alternatives = True mlist.convert_html_to_plaintext = False + mlist.filter_action = FilterAction.discard mlist.filter_content = False # Digest related variables mlist.digestable = True @@ -132,7 +132,7 @@ from: .*@uplinkpro.com # Max autoresponses per day. A mapping between addresses and a # 2-tuple of the date of the last autoresponse and the number of # autoresponses sent on that date. - mlist.subject_prefix = _('[$mlist.real_name] ') + mlist.subject_prefix = _('[$mlist.display_name] ') mlist.header_uri = None mlist.footer_uri = 'mailman:///$listname/$language/footer-generic.txt' # Set this to Never if the list's preferred language uses us-ascii, @@ -152,16 +152,15 @@ from: .*@uplinkpro.com mlist.autoresponse_postings_text = '' mlist.autorespond_requests = ResponseAction.none mlist.autoresponse_request_text = '' - mlist.autoresponse_grace_period = datetime.timedelta(days=90) + mlist.autoresponse_grace_period = timedelta(days=90) # Bounces mlist.forward_unrecognized_bounces_to = ( UnrecognizedBounceDisposition.administrators) mlist.process_bounces = True mlist.bounce_score_threshold = 5.0 - mlist.bounce_info_stale_after = datetime.timedelta(days=7) + mlist.bounce_info_stale_after = timedelta(days=7) mlist.bounce_you_are_disabled_warnings = 3 - mlist.bounce_you_are_disabled_warnings_interval = ( - datetime.timedelta(days=7)) + mlist.bounce_you_are_disabled_warnings_interval = timedelta(days=7) mlist.bounce_notify_owner_on_disable = True mlist.bounce_notify_owner_on_removal = True # This holds legacy member related information. It's keyed by the @@ -217,6 +216,11 @@ from: .*@uplinkpro.com # The default pipeline to send accepted messages through to the # mailing list's members. mlist.posting_pipeline = 'default-posting-pipeline' + # The processing chain that messages posted to this mailing list's + # -owner address gets processed by. + mlist.owner_chain = 'default-owner-chain' + # The default pipeline to send -owner email through. + mlist.owner_pipeline = 'default-owner-pipeline' def match(self, mailing_list, styles): """See `IStyle`.""" diff --git a/src/mailman/templates/en/archidxentry.html b/src/mailman/templates/en/archidxentry.html deleted file mode 100644 index 1927ae7fe..000000000 --- a/src/mailman/templates/en/archidxentry.html +++ /dev/null @@ -1,4 +0,0 @@ -<LI><A HREF="$filename">$subject -</A><A NAME="$sequence"> </A> -<I>$author -</I> diff --git a/src/mailman/templates/en/archidxfoot.html b/src/mailman/templates/en/archidxfoot.html deleted file mode 100644 index 6a43546ea..000000000 --- a/src/mailman/templates/en/archidxfoot.html +++ /dev/null @@ -1,21 +0,0 @@ - </ul> - <p> - <a name="end"><b>Last message date:</b></a> - <i>$lastdate</i><br> - <b>Archived on:</b> <i>$archivedate</i> - <p> - <ul> - <li> <b>Messages sorted by:</b> - $thread_ref - $subject_ref - $author_ref - $date_ref - <li><b><a href="$listinfo">More info on this list... - </a></b></li> - </ul> - <p> - <hr> - <i>This archive was generated by - Pipermail $version.</i> - </BODY> -</HTML> diff --git a/src/mailman/templates/en/archidxhead.html b/src/mailman/templates/en/archidxhead.html deleted file mode 100644 index 70a7558d7..000000000 --- a/src/mailman/templates/en/archidxhead.html +++ /dev/null @@ -1,24 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN"> -<HTML> - <HEAD> - <title>The $listname $archive Archive by $archtype</title> - <META NAME="robots" CONTENT="noindex,follow"> - $encoding - </HEAD> - <BODY BGCOLOR="#ffffff"> - <a name="start"></A> - <h1>$archive Archives by $archtype</h1> - <ul> - <li> <b>Messages sorted by:</b> - $thread_ref - $subject_ref - $author_ref - $date_ref - - <li><b><a href="$listinfo">More info on this list... - </a></b></li> - </ul> - <p><b>Starting:</b> <i>$firstdate</i><br> - <b>Ending:</b> <i>$lastdate</i><br> - <b>Messages:</b> $size<p> - <ul> diff --git a/src/mailman/templates/en/archlistend.html b/src/mailman/templates/en/archlistend.html deleted file mode 100644 index 9bc052ddb..000000000 --- a/src/mailman/templates/en/archlistend.html +++ /dev/null @@ -1 +0,0 @@ - </table> diff --git a/src/mailman/templates/en/archliststart.html b/src/mailman/templates/en/archliststart.html deleted file mode 100644 index cdf5d17c4..000000000 --- a/src/mailman/templates/en/archliststart.html +++ /dev/null @@ -1,4 +0,0 @@ - <table border=3> - <tr><td>Archive</td> - <td>View by:</td> - <td>Downloadable version</td></tr> diff --git a/src/mailman/templates/en/archtoc.html b/src/mailman/templates/en/archtoc.html deleted file mode 100644 index 4dcaf5a50..000000000 --- a/src/mailman/templates/en/archtoc.html +++ /dev/null @@ -1,20 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN"> -<HTML> - <HEAD> - <title>The $listname Archives</title> - <META NAME="robots" CONTENT="noindex,follow"> - $meta - </HEAD> - <BODY BGCOLOR="#ffffff"> - <h1>The $listname Archives </h1> - <p> - You can get <a href="$listinfo">more information about this list</a> - or you can <a href="$fullarch">download the full raw archive</a> - ($size). - </p> - $noarchive_msg - $archive_listing_start - $archive_listing - $archive_listing_end - </BODY> - </HTML> diff --git a/src/mailman/templates/en/archtocentry.html b/src/mailman/templates/en/archtocentry.html deleted file mode 100644 index e2a6d2e37..000000000 --- a/src/mailman/templates/en/archtocentry.html +++ /dev/null @@ -1,12 +0,0 @@ - - <tr> - <td>$archivelabel:</td> - <td> - <A href="$archive/thread.html">[ Thread ]</a> - <A href="$archive/subject.html">[ Subject ]</a> - <A href="$archive/author.html">[ Author ]</a> - <A href="$archive/date.html">[ Date ]</a> - </td> - $textlink - </tr> - diff --git a/src/mailman/templates/en/archtocnombox.html b/src/mailman/templates/en/archtocnombox.html deleted file mode 100644 index 5989aa53d..000000000 --- a/src/mailman/templates/en/archtocnombox.html +++ /dev/null @@ -1,18 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN"> -<HTML> - <HEAD> - <title>The $listname Archives</title> - <META NAME="robots" CONTENT="noindex,follow"> - $meta - </HEAD> - <BODY BGCOLOR="#ffffff"> - <h1>The $listname Archives </h1> - <p> - You can get <a href="$listinfo">more information about this list</a>. - </p> - $noarchive_msg - $archive_listing_start - $archive_listing - $archive_listing_end - </BODY> - </HTML> diff --git a/src/mailman/templates/en/footer-generic.txt b/src/mailman/templates/en/footer-generic.txt index ef76e4986..d31e885f0 100644 --- a/src/mailman/templates/en/footer-generic.txt +++ b/src/mailman/templates/en/footer-generic.txt @@ -1,4 +1,4 @@ _______________________________________________ -$list_name mailing list +$display_name mailing list $fqdn_listname ${listinfo_uri} diff --git a/src/mailman/templates/en/masthead.txt b/src/mailman/templates/en/masthead.txt index 01cd6afec..5d4cc9696 100644 --- a/src/mailman/templates/en/masthead.txt +++ b/src/mailman/templates/en/masthead.txt @@ -1,4 +1,4 @@ -Send $real_name mailing list submissions to +Send $display_name mailing list submissions to $got_list_email To subscribe or unsubscribe via the World Wide Web, visit @@ -10,4 +10,4 @@ You can reach the person managing the list at $got_owner_email When replying, please edit your Subject line so it is more specific than -"Re: Contents of $real_name digest..." +"Re: Contents of $display_name digest..." diff --git a/src/mailman/templates/en/postack.txt b/src/mailman/templates/en/postack.txt index a2d6e972b..7f5836344 100644 --- a/src/mailman/templates/en/postack.txt +++ b/src/mailman/templates/en/postack.txt @@ -2,7 +2,7 @@ Your message entitled $subject -was successfully received by the $listname mailing list. +was successfully received by the $display_name mailing list. List info page: $listinfo_url Your preferences: $optionsurl diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 58c72d6d9..032b028a9 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -1,3 +1,4 @@ +# Copyright (C) 2008-2012 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -22,6 +23,7 @@ __metaclass__ = type __all__ = [ 'LogFileMark', 'TestableMaster', + 'body_line_iterator', 'call_api', 'digest_mbox', 'event_subscribers', @@ -412,3 +414,19 @@ class LogFileMark: with open(self._filename) as fp: fp.seek(self._filepos) return fp.readline() + + + +# In Python 2.6, body_line_iterator() uses a cStringIO.StringIO() which cannot +# handle unicode. In Python 2.7 this works fine. I hate version checks but +# this is the easiest way to handle it. OTOH, we could just use the manual +# way for all Python versions instead. +import sys +if sys.hexversion >= 0x2070000: + from email.iterators import body_line_iterator +else: + def body_line_iterator(msg, decode=False): + payload = msg.get_payload(decode=decode) + bytes_payload = payload.encode('utf-8') + for line in bytes_payload.splitlines(): + yield line diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 04ab8f91f..41ef86935 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -17,6 +17,14 @@ """Mailman test layers.""" +# XXX 2012-03-23 BAW: Layers really really suck. For example, the +# test_owners_get_email() test requires that both the SMTPLayer and LMTPLayer +# be set up, but there's apparently no way to do that and make zope.testing +# happy. This causes no tests failures, but it does cause errors at the end +# of the full test run. For now, I'll ignore that, but I do want to +# eventually get rid of the zope.test* dependencies and use something like +# testresources or some such. + from __future__ import absolute_import, unicode_literals __metaclass__ = type diff --git a/src/mailman/testing/mailman-fr.mo b/src/mailman/testing/mailman-fr.mo Binary files differindex df721cb0f..5758baf60 100644 --- a/src/mailman/testing/mailman-fr.mo +++ b/src/mailman/testing/mailman-fr.mo diff --git a/src/mailman/testing/mailman-fr.po b/src/mailman/testing/mailman-fr.po index 5f8c910e2..ca4ce8c32 100644 --- a/src/mailman/testing/mailman-fr.po +++ b/src/mailman/testing/mailman-fr.po @@ -17,7 +17,7 @@ msgstr "" #: templates/en/masthead.txt:1 msgid "" -"Send $real_name mailing list submissions to\n" +"Send $display_name mailing list submissions to\n" "\t$got_list_email\n" "\n" "To subscribe or unsubscribe via the World Wide Web, visit\n" @@ -29,9 +29,9 @@ msgid "" "\t$got_owner_email\n" "\n" "When replying, please edit your Subject line so it is more specific than\n" -"\"Re: Contents of $real_name digest...\"" +"\"Re: Contents of $display_name digest...\"" msgstr "" -"Envoyez vos messages pour la liste $real_name à\n" +"Envoyez vos messages pour la liste $display_name à\n" "\t$got_list_email\n" "\n" "Pour vous (dés)abonner par le web, consultez\n" @@ -45,11 +45,11 @@ msgstr "" "\t$got_owner_email\n" "\n" "Si vous répondez, n'oubliez pas de changer l'objet du message afin\n" -"qu'il soit plus spécifique que « Re: Contenu du groupe de $real_name... »" +"qu'il soit plus spécifique que « Re: Contenu du groupe de $display_name... »" #: Mailman/Handlers/ToDigest.py:159 -msgid "$realname Digest, Vol $volume, Issue $issue" -msgstr "Groupe $realname, Vol. $volume, Parution $issue" +msgid "$mlist.display_name Digest, Vol $volume, Issue $digest_number" +msgstr "Groupe $mlist.display_name, Vol $volume, Parution $digest_number" #: Mailman/Handlers/ToDigest.py:205 msgid "digest header" diff --git a/src/mailman/testing/mailman-xx.mo b/src/mailman/testing/mailman-xx.mo Binary files differindex ecbad0364..bfc2e0845 100644 --- a/src/mailman/testing/mailman-xx.mo +++ b/src/mailman/testing/mailman-xx.mo diff --git a/src/mailman/testing/mailman-xx.po b/src/mailman/testing/mailman-xx.po index e2ad28c33..c6543d3d5 100644 --- a/src/mailman/testing/mailman-xx.po +++ b/src/mailman/testing/mailman-xx.po @@ -13,5 +13,5 @@ msgstr "" "Generated-By: pygettext.py 1.3\n" #: src/mailman/app/bounces.py:227 -msgid "$mlist.real_name mailing list probe message" -msgstr "ailing-may ist-lay $mlist.real_name obe-pray essage-may" +msgid "$mlist.display_name mailing list probe message" +msgstr "ailing-may ist-lay $mlist.display_name obe-pray essage-may" diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 526093572..b7e80ff02 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -22,6 +22,9 @@ #class: mailman.database.postgresql.PostgreSQLDatabase #url: postgres://barry:barry@localhost/mailman +[mailman] +site_owner: noreply@example.com + [mta] smtp_port: 9025 lmtp_port: 9024 @@ -71,10 +74,6 @@ enable: yes base_url: http://go.mail-archive.dev/ recipient: archive@mail-archive.dev -[archiver.pipermail] -enable: yes -base_url: http://www.example.com/pipermail/$listname - [archiver.mhonarc] enable: yes command: /bin/echo "/usr/bin/mhonarc -add -dbfile $PRIVATE_ARCHIVE_FILE_DIR/${listname}.mbox/mhonarc.db -outdir $VAR_DIR/mhonarc/${listname} -stderr $LOG_DIR/mhonarc -stdout $LOG_DIR/mhonarc -spammode -umask 022" diff --git a/src/mailman/utilities/datetime.py b/src/mailman/utilities/datetime.py index 96d14fce5..3f451efaf 100644 --- a/src/mailman/utilities/datetime.py +++ b/src/mailman/utilities/datetime.py @@ -28,9 +28,12 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'DateFactory', + 'RFC822_DATE_FMT', + 'UTC', 'factory', 'now', 'today', + 'utc', ] @@ -39,6 +42,30 @@ import datetime from mailman.testing import layers +# Python always sets the locale to 'C' locale unless the user explicitly calls +# locale.setlocale(locale.LC_ALL, ''). Since we never do this in Mailman (and +# no library better do it either!) this will safely give us expected RFC 5322 +# Date headers. +RFC822_DATE_FMT = '%a, %d %b %Y %H:%M:%S %z' + + + +# Definition of UTC timezone, taken from +# http://docs.python.org/library/datetime.html +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + def tzname(self, dt): + return 'UTC' + def dst(self, dt): + return ZERO + +utc = UTC() +_missing = object() + + class DateFactory: """A factory for today() and now() that works with testing.""" @@ -47,12 +74,21 @@ class DateFactory: predictable_now = None predictable_today = None - def now(self, tz=None): + def now(self, tz=_missing, strip_tzinfo=True): # We can't automatically fast-forward because some tests require us to # stay on the same day for a while, e.g. autorespond.txt. - return (self.predictable_now - if layers.is_testing() - else datetime.datetime.now(tz)) + if tz is _missing: + tz = utc + # Storm cannot yet handle datetimes with tz suffixes. Assume we're + # using UTC datetimes everywhere, so set the tzinfo to None. This + # does *not* change the actual time values. LP: #280708 + tz_now = (self.predictable_now + if layers.is_testing() + else datetime.datetime.now(tz)) + return (tz_now.replace(tzinfo=None) + if strip_tzinfo + else tz_now) + def today(self): return (self.predictable_today @@ -61,7 +97,8 @@ class DateFactory: @classmethod def reset(cls): - cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23) + cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23, + tzinfo=utc) cls.predictable_today = cls.predictable_now.date() @classmethod diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index d31a3892f..f77d86e9a 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -56,6 +56,7 @@ TYPES = dict( # Attribute names in Mailman 2 which are renamed in Mailman 3. NAME_MAPPINGS = dict( host_name='mail_host', + real_name='display_name', ) diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index 2ce25ddc6..58a51e61b 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -49,11 +49,11 @@ class TestBasicImport(unittest.TestCase): def _import(self): import_config_pck(self._mlist, self._pckdict) - def test_real_name(self): - # The mlist.real_name gets set. - self.assertEqual(self._mlist.real_name, 'Blank') + def test_display_name(self): + # The mlist.display_name gets set from the old list's real_name. + self.assertEqual(self._mlist.display_name, 'Blank') self._import() - self.assertEqual(self._mlist.real_name, 'Test') + self.assertEqual(self._mlist.display_name, 'Test') def test_mail_host(self): # The mlist.mail_host gets set. diff --git a/src/mailman/version.py b/src/mailman/version.py index 8f13f8f51..a75bea025 100644 --- a/src/mailman/version.py +++ b/src/mailman/version.py @@ -18,8 +18,8 @@ """Mailman version strings.""" # Mailman version. -VERSION = '3.0.0a8+' -CODENAME = "The Twilight Zone" +VERSION = '3.0.0b1+' +CODENAME = "Freeze" # And as a hex number in the manner of PY_VERSION_HEX. ALPHA = 0xa @@ -34,7 +34,7 @@ MINOR_REV = 0 MICRO_REV = 0 REL_LEVEL = BETA # At most 15 beta releases! -REL_SERIAL = 1 +REL_SERIAL = 2 HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) |