# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # # GNU Mailman is free software: you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see . """HyperArch: Pipermail archiving for Mailman - The Dragon De Monsyne TODO: - Should be able to force all HTML to be regenerated next time the archive is run, in case a template is changed. - Run a command to generate tarball of html archives for downloading (probably in the 'update_dirty_archives' method). """ import os import re import sys import gzip import time import errno import urllib import logging import binascii from email.Charset import Charset from email.Errors import HeaderParseError from email.Header import decode_header, make_header from flufl.lock import Lock, TimeOutError from lazr.config import as_boolean from string import Template from zope.component import getUtility from mailman.Archiver import HyperDatabase from mailman.Archiver import pipermail from mailman.config import config from mailman.core.i18n import _, ctime from mailman.interfaces.listmanager import IListManager from mailman.utilities.i18n import find from mailman.utilities.string import uncanonstr, websafe log = logging.getLogger('mailman.error') EMPTYSTRING = '' NL = '\n' # MacOSX has a default stack size that is too small for deeply recursive # regular expressions. We see this as crashes in the Python test suite when # running test_re.py and test_sre.py. The fix is to set the stack limit to # 2048; the general recommendation is to do in the shell before running the # test suite. But that's inconvenient for a daemon like the qrunner. # # AFAIK, this problem only affects the archiver, so we're adding this work # around to this file (it'll get imported by the bundled pipermail or by the # bin/arch script. We also only do this on darwin, a.k.a. MacOSX. if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) def html_quote(s, langcode=None): repls = ( ('&', '&'), ("<", '<'), (">", '>'), ('"', '"')) for thing, repl in repls: s = s.replace(thing, repl) return uncanonstr(s, langcode) def url_quote(s): return urllib.quote(s) def null_to_space(s): return s.replace('\000', ' ') def sizeof(filename, lang): try: size = os.path.getsize(filename) except OSError, e: # ENOENT can happen if the .mbox file was moved away or deleted, and # an explicit mbox file name was given to bin/arch. if e.errno <> errno.ENOENT: raise return _('size not available') if size < 1000: with _.using(lang.code): out = _(' %(size)i bytes ') return out elif size < 1000000: return ' %d KB ' % (size / 1000) # GB?? :-) return ' %d MB ' % (size / 1000000) html_charset = '' def CGIescape(arg, lang=None): if isinstance(arg, unicode): s = websafe(arg) else: s = websafe(str(arg)) return uncanonstr(s.replace('"', '"'), lang.code) # Parenthesized human name paren_name_pat = re.compile(r'([(].*[)])') # Subject lines preceded with 'Re:' REpat = re.compile( r"\s*RE\s*(\[\d+\]\s*)?:\s*", re.IGNORECASE) # E-mail addresses and URLs in text emailpat = re.compile(r'([-+,.\w]+@[-+.\w]+)') # Argh! This pattern is buggy, and will choke on URLs with GET parameters. urlpat = re.compile(r'(\w+://[^>)\s]+)') # URLs in text # Blank lines blankpat = re.compile(r'^\s*$') # Starting directive htmlpat = re.compile(r'^\s*\s*$', re.IGNORECASE) # Ending directive nohtmlpat = re.compile(r'^\s*\s*$', re.IGNORECASE) # Match quoted text quotedpat = re.compile(r'^([>|:]|>)+') # Like Utils.maketext() but with caching to improve performance. # # _templatefilepathcache is used to associate a (templatefile, lang, listname) # key with the file system path to a template file. This path is the one that # the Utils.findtext() function has computed is the one to match the values in # the key tuple. # # _templatecache associate a file system path as key with the text # returned after processing the contents of that file by Utils.findtext() # # We keep two caches to reduce the amount of template text kept in memory, # since the _templatefilepathcache is a many->one mapping and _templatecache # is a one->one mapping. Imagine 1000 lists all using the same default # English template. _templatefilepathcache = {} _templatecache = {} def quick_maketext(templatefile, dict=None, lang=None, mlist=None): if mlist is None: listname = '' else: listname = mlist.fqdn_listname if lang is None: if mlist is None: lang = config.mailman.default_language else: lang = mlist.preferred_language cachekey = (templatefile, lang, listname) filepath = _templatefilepathcache.get(cachekey) if filepath: template = _templatecache.get(filepath) if filepath is None or template is None: # Use the basic maketext, with defaults to get the raw template template, filepath = find(templatefile, mailing_list=mlist, language=lang.code) _templatefilepathcache[cachekey] = filepath _templatecache[filepath] = template # Copied from Utils.maketext() text = template if dict is not None: try: try: text = Template(template).safe_substitute(**dict) except UnicodeError: # Try again after coercing the template to unicode utemplate = unicode(template, lang.charset, 'replace') text = Template(utemplate).safe_substitute(**dict) except (TypeError, ValueError): # The template is really screwed up pass # Make sure the text is in the given character set, or html-ify any bogus # characters. return uncanonstr(text, lang.code) # Note: I'm overriding most, if not all of the pipermail Article class # here -ddm # The Article class encapsulates a single posting. The attributes are: # # sequence : Sequence number, unique for each article in a set of archives # subject : Subject # datestr : The posting date, in human-readable format # date : The posting date, in purely numeric format # fromdate : The posting date, in `unixfrom' format # headers : Any other headers of interest # author : The author's name (and possibly organization) # email : The author's e-mail address # msgid : A unique message ID # in_reply_to : If !="", this is the msgid of the article being replied to # references: A (possibly empty) list of msgid's of earlier articles in # the thread # body : A list of strings making up the message body class Article(pipermail.Article): __super_init = pipermail.Article.__init__ __super_set_date = pipermail.Article._set_date _last_article_time = time.time() def __init__(self, message=None, sequence=0, keepHeaders=[], lang=config.mailman.default_language, mlist=None): self.__super_init(message, sequence, keepHeaders) self.prev = None self.next = None # Trim Re: from the subject line i = 0 while i != -1: result = REpat.match(self.subject) if result: i = result.end(0) self.subject = self.subject[i:] else: i = -1 # Useful to keep around self._lang = lang self._mlist = mlist if as_boolean(config.archiver.pipermail.obscure_email_addresses): # Avoid i18n side-effects. Note that the language for this # article (for this list) could be different from the site-wide # preferred language, so we need to ensure no side-effects will # occur. Think what happens when executing bin/arch. with _.using(lang.code): if self.author == self.email: self.author = self.email = re.sub('@', _(' at '), self.email) else: self.email = re.sub('@', _(' at '), self.email) # Snag the content-* headers. RFC 1521 states that their values are # case insensitive. ctype = message.get('Content-Type', 'text/plain') cenc = message.get('Content-Transfer-Encoding', '') self.ctype = ctype.lower() self.cenc = cenc.lower() self.decoded = {} cset = mlist.preferred_language.charset cset_out = Charset(cset).output_charset or cset charset = message.get_content_charset(cset_out) if charset: charset = charset.lower().strip() if charset[0]=='"' and charset[-1]=='"': charset = charset[1:-1] if charset[0]=="'" and charset[-1]=="'": charset = charset[1:-1] try: body = message.get_payload(decode=True) except binascii.Error: body = None if body and charset != self._lang.charset: # decode body try: body = unicode(body, charset) except (UnicodeError, LookupError): body = None if body: self.body = [l + "\n" for l in body.splitlines()] self.decode_headers() def __getstate__(self): d = self.__dict__.copy() # We definitely don't want to pickle the MailList instance, so just # pickle a reference to it. if d.has_key('_mlist'): mlist = d['_mlist'] del d['_mlist'] else: mlist = None if mlist: d['__listname'] = self._mlist.fqdn_listname else: d['__listname'] = None # Delete a few other things we don't want in the pickle for attr in ('prev', 'next', 'body'): if d.has_key(attr): del d[attr] d['body'] = [] return d def __setstate__(self, d): # For loading older Articles via pickle. All this stuff was added # when Simone Piunni and Tokio Kikuchi i18n'ified Pipermail. See SF # patch #594771. self.__dict__ = d listname = d.get('__listname') if listname: del d['__listname'] d['_mlist'] = getUtility(IListManager).get(listname) if not d.has_key('_lang'): if hasattr(self, '_mlist'): self._lang = self._mlist.preferred_language else: self._lang = config.mailman.default_language if not d.has_key('cenc'): self.cenc = None if not d.has_key('decoded'): self.decoded = {} def setListIfUnset(self, mlist): if getattr(self, '_mlist', None) is None: self._mlist = mlist def quote(self, buf): return html_quote(buf, self._lang.code) def decode_headers(self): """MIME-decode headers. If the email, subject, or author attributes contain non-ASCII characters using the encoded-word syntax of RFC 2047, decoded versions of those attributes are placed in the self.decoded (a dictionary). If the list's charset differs from the header charset, an attempt is made to decode the headers as Unicode. If that fails, they are left undecoded. """ author = self.decode_charset(self.author) subject = self.decode_charset(self.subject) if author: self.decoded['author'] = author email = self.decode_charset(self.email) if email: self.decoded['email'] = email if subject: if as_boolean(config.archiver.pipermail.obscure_email_addresses): with _.using(self._lang.code): atmark = _(' at ') subject = re.sub(r'([-+,.\w]+)@([-+.\w]+)', '\g<1>' + atmark + '\g<2>', subject) self.decoded['subject'] = subject self.decoded['stripped'] = self.strip_subject(subject or self.subject) def strip_subject(self, subject): # Strip subject_prefix and Re: for subject sorting # This part was taken from CookHeaders.py (TK) prefix = self._mlist.subject_prefix.strip() if prefix: prefix_pat = re.escape(prefix) prefix_pat = '%'.join(prefix_pat.split(r'\%')) prefix_pat = re.sub(r'%\d*d', r'\s*\d+\s*', prefix_pat) subject = re.sub(prefix_pat, '', subject) subject = subject.lstrip() strip_pat = re.compile('^((RE|AW|SV|VS)(\[\d+\])?:\s*)+', re.I) stripped = strip_pat.sub('', subject) return stripped def decode_charset(self, field): # TK: This function was rewritten for unifying to Unicode. # Convert 'field' into Unicode one line string. try: pairs = decode_header(field) ustr = make_header(pairs).__unicode__() except (LookupError, UnicodeError, ValueError, HeaderParseError): # assume list's language cset = self._mlist.preferred_language.charset if cset == 'us-ascii': cset = 'iso-8859-1' # assume this for English list ustr = unicode(field, cset, 'replace') return u''.join(ustr.splitlines()) def as_html(self): d = self.__dict__.copy() # Avoid i18n side-effects with _.using(self._lang.code): d["prev"], d["prev_wsubj"] = self._get_prev() d["next"], d["next_wsubj"] = self._get_next() d["email_html"] = self.quote(self.email) d["title"] = self.quote(self.subject) d["subject_html"] = self.quote(self.subject) d["subject_url"] = url_quote(self.subject) d["in_reply_to_url"] = url_quote(self.in_reply_to) if as_boolean(config.archiver.pipermail.obscure_email_addresses): # Point the mailto url back to the list author = re.sub('@', _(' at '), self.author) emailurl = self._mlist.posting_address else: author = self.author emailurl = self.email d["author_html"] = self.quote(author) d["email_url"] = url_quote(emailurl) d["datestr_html"] = self.quote(ctime(int(self.date))) d["body"] = self._get_body() d['listurl'] = self._mlist.script_url('listinfo') d['listname'] = self._mlist.real_name d['encoding'] = '' charset = self._lang.charset d["encoding"] = html_charset % charset self._add_decoded(d) return quick_maketext( 'article.html', d, lang=self._lang, mlist=self._mlist) def _get_prev(self): """Return the href and subject for the previous message""" if self.prev: subject = self._get_subject_enc(self.prev) prev = ('' % (url_quote(self.prev.filename))) prev_wsubj = ('
  • ' + _('Previous message (by thread):') + ' %s\n
    • ' % (url_quote(self.prev.filename), self.quote(subject))) else: prev = prev_wsubj = "" return prev, prev_wsubj def _get_subject_enc(self, art): """Return the subject of art, decoded if possible. If the charset of the current message and art match and the article's subject is encoded, decode it. """ return art.decoded.get('subject', art.subject) def _get_next(self): """Return the href and subject for the previous message""" if self.next: subject = self._get_subject_enc(self.next) next = ('' % (url_quote(self.next.filename))) next_wsubj = ('
  • ' + _('Next message (by thread):') + ' %s\n
    • ' % (url_quote(self.next.filename), self.quote(subject))) else: next = next_wsubj = "" return next, next_wsubj _rx_quote = re.compile('=([A-F0-9][A-F0-9])') _rx_softline = re.compile('=[ \t]*$') def _get_body(self): """Return the message body ready for HTML, decoded if necessary""" try: body = self.html_body except AttributeError: body = self.body return null_to_space(EMPTYSTRING.join(body)) def _add_decoded(self, d): """Add encoded-word keys to HTML output""" for src, dst in (('author', 'author_html'), ('email', 'email_html'), ('subject', 'subject_html'), ('subject', 'title')): if self.decoded.has_key(src): d[dst] = self.quote(self.decoded[src]) def as_text(self): d = self.__dict__.copy() # We need to guarantee a valid From_ line, even if there are # bososities in the headers. if not d.get('fromdate', '').strip(): d['fromdate'] = time.ctime(time.time()) if not d.get('email', '').strip(): d['email'] = 'bogus@does.not.exist.com' if not d.get('datestr', '').strip(): d['datestr'] = time.ctime(time.time()) # headers = ['From %(email)s %(fromdate)s', 'From: %(email)s (%(author)s)', 'Date: %(datestr)s', 'Subject: %(subject)s'] if d['_in_reply_to']: headers.append('In-Reply-To: %(_in_reply_to)s') if d['_references']: headers.append('References: %(_references)s') if d['_message_id']: headers.append('Message-ID: %(_message_id)s') body = EMPTYSTRING.join(self.body) cset = self._lang.charset # Coerce the body to Unicode and replace any invalid characters. if not isinstance(body, unicode): body = unicode(body, cset, 'replace') if as_boolean(config.archiver.pipermail.obscure_email_addresses): with _.using(self._lang.code): atmark = _(' at ') body = re.sub(r'([-+,.\w]+)@([-+.\w]+)', '\g<1>' + atmark + '\g<2>', body) # Return body to character set of article. body = body.encode(cset, 'replace') return NL.join(headers) % d + '\n\n' + body + '\n' def _set_date(self, message): self.__super_set_date(message) self.fromdate = time.ctime(int(self.date)) def loadbody_fromHTML(self,fileobj): self.body = [] begin = 0 while 1: line = fileobj.readline() if not line: break if not begin: if line.strip() == '': begin = 1 continue if line.strip() == '': break self.body.append(line) def finished_update_article(self): self.body = [] try: del self.html_body except AttributeError: pass class HyperArchive(pipermail.T): __super_init = pipermail.T.__init__ __super_update_archive = pipermail.T.update_archive __super_update_dirty_archives = pipermail.T.update_dirty_archives __super_add_article = pipermail.T.add_article # some defaults DIRMODE = 02775 FILEMODE = 0660 VERBOSE = 0 DEFAULTINDEX = 'thread' ARCHIVE_PERIOD = 'month' THREADLAZY = 0 THREADLEVELS = 3 ALLOWHTML = 1 # "Lines between " handled as is. SHOWHTML = 0 # Eg, nuke leading whitespace in html manner. IQUOTES = 1 # Italicize quoted text. SHOWBR = 0 # Add
    onto every line def __init__(self, maillist): # can't init the database while other processes are writing to it! dir = maillist.archive_dir() db = HyperDatabase.HyperDatabase(dir, maillist) self.__super_init(dir, reload=1, database=db) self.maillist = maillist self._lock_file = None self.lang = maillist.preferred_language self.charset = maillist.preferred_language.charset if hasattr(self.maillist,'archive_volume_frequency'): if self.maillist.archive_volume_frequency == 0: self.ARCHIVE_PERIOD='year' elif self.maillist.archive_volume_frequency == 2: self.ARCHIVE_PERIOD='quarter' elif self.maillist.archive_volume_frequency == 3: self.ARCHIVE_PERIOD='week' elif self.maillist.archive_volume_frequency == 4: self.ARCHIVE_PERIOD='day' else: self.ARCHIVE_PERIOD='month' yre = r'(?P[0-9]{4,4})' mre = r'(?P[01][0-9])' dre = r'(?P[0123][0-9])' self._volre = { 'year': '^' + yre + '$', 'quarter': '^' + yre + r'q(?P[1234])$', 'month': '^' + yre + r'-(?P[a-zA-Z]+)$', 'week': r'^Week-of-Mon-' + yre + mre + dre, 'day': '^' + yre + mre + dre + '$' } def _makeArticle(self, msg, sequence): return Article(msg, sequence, lang=self.maillist.preferred_language, mlist=self.maillist) def html_foot(self): mlist = self.maillist # Convenience def quotetime(s): return html_quote(ctime(s), self.lang.code) # Avoid i18n side-effects with _.using(mlist.preferred_language.code): d = {"lastdate": quotetime(self.lastdate), "archivedate": quotetime(self.archivedate), "listinfo": mlist.script_url('listinfo'), "version": self.version, } i = {"thread": _("thread"), "subject": _("subject"), "author": _("author"), "date": _("date") } for t in i.keys(): cap = t[0].upper() + t[1:] if self.type == cap: d["%s_ref" % (t)] = "" else: d["%s_ref" % (t)] = ('[ %s ]' % (t, i[t])) return quick_maketext( 'archidxfoot.html', d, mlist=mlist) def html_head(self): mlist = self.maillist # Convenience def quotetime(s): return html_quote(ctime(s), self.lang.code) # Avoid i18n side-effects with _.using(mlist.preferred_language.code): d = {"listname": html_quote(mlist.real_name, self.lang.code), "archtype": self.type, "archive": self.volNameToDesc(self.archive), "listinfo": mlist.script_url('listinfo'), "firstdate": quotetime(self.firstdate), "lastdate": quotetime(self.lastdate), "size": self.size, } i = {"thread": _("thread"), "subject": _("subject"), "author": _("author"), "date": _("date"), } for t in i.keys(): cap = t[0].upper() + t[1:] if self.type == cap: d["%s_ref" % (t)] = "" d["archtype"] = i[t] else: d["%s_ref" % (t)] = ('[ %s ]' % (t, i[t])) if self.charset: d["encoding"] = html_charset % self.charset else: d["encoding"] = "" return quick_maketext( 'archidxhead.html', d, mlist=mlist) def html_TOC(self): mlist = self.maillist listname = mlist.fqdn_listname mbox = os.path.join(mlist.archive_dir()+'.mbox', listname+'.mbox') d = {"listname": mlist.real_name, "listinfo": mlist.script_url('listinfo'), "fullarch": '../%s.mbox/%s.mbox' % (listname, listname), "size": sizeof(mbox, mlist.preferred_language), 'meta': '', } # Avoid i18n side-effects with _.using(mlist.preferred_language.code): if not self.archives: d["noarchive_msg"] = _( '

    Currently, there are no archives.

    ') d["archive_listing_start"] = "" d["archive_listing_end"] = "" d["archive_listing"] = "" else: d["noarchive_msg"] = "" d["archive_listing_start"] = quick_maketext( 'archliststart.html', lang=mlist.preferred_language, mlist=mlist) d["archive_listing_end"] = quick_maketext( 'archlistend.html', mlist=mlist) accum = [] for a in self.archives: accum.append(self.html_TOC_entry(a)) d["archive_listing"] = EMPTYSTRING.join(accum) # The TOC is always in the charset of the list's preferred language d['meta'] += html_charset % mlist.preferred_language.charset # The site can disable public access to the mbox file. if as_boolean(config.archiver.pipermail.public_mbox): template = 'archtoc.html' else: template = 'archtocnombox.html' return quick_maketext(template, d, mlist=mlist) def html_TOC_entry(self, arch): # Check to see if the archive is gzip'd or not txtfile = os.path.join(self.maillist.archive_dir(), arch + '.txt') gzfile = txtfile + '.gz' # which exists? .txt.gz first, then .txt if os.path.exists(gzfile): file = gzfile url = arch + '.txt.gz' templ = '[ ' + _('Gzip\'d Text%(sz)s') \ + ']' elif os.path.exists(txtfile): file = txtfile url = arch + '.txt' templ = '[ ' + _('Text%(sz)s') + ']' else: # neither found? file = None # in Python 1.5.2 we have an easy way to get the size if file: textlink = templ % { 'url': url, 'sz' : sizeof(file, self.maillist.preferred_language) } else: # there's no archive file at all... hmmm. textlink = '' return quick_maketext( 'archtocentry.html', {'archive': arch, 'archivelabel': self.volNameToDesc(arch), 'textlink': textlink }, mlist=self.maillist) def GetArchLock(self): if self._lock_file: return 1 self._lock_file = Lock( os.path.join(config.LOCK_DIR, self.maillist.fqdn_listname + '-arch.lock')) try: self._lock_file.lock(timeout=0.5) except TimeOutError: return 0 return 1 def DropArchLock(self): if self._lock_file: self._lock_file.unlock(unconditionally=1) self._lock_file = None def processListArch(self): name = self.maillist.ArchiveFileName() wname= name+'.working' ename= name+'.err_unarchived' try: os.stat(name) except (IOError,os.error): #no archive file, nothin to do -ddm return #see if arch is locked here -ddm if not self.GetArchLock(): #another archiver is running, nothing to do. -ddm return #if the working file is still here, the archiver may have # crashed during archiving. Save it, log an error, and move on. try: wf = open(wname) log.error('Archive working file %s present. ' 'Check %s for possibly unarchived msgs', wname, ename) omask = os.umask(007) try: ef = open(ename, 'a+') finally: os.umask(omask) ef.seek(1,2) if ef.read(1) <> '\n': ef.write('\n') ef.write(wf.read()) ef.close() wf.close() os.unlink(wname) except IOError: pass os.rename(name,wname) archfile = open(wname) self.processUnixMailbox(archfile) archfile.close() os.unlink(wname) self.DropArchLock() def get_filename(self, article): return '%06i.html' % (article.sequence,) def get_archives(self, article): """Return a list of indexes where the article should be filed. A string can be returned if the list only contains one entry, and the empty list is legal.""" res = self.dateToVolName(float(article.date)) self.message(_("figuring article archives\n")) self.message(res + "\n") return res def volNameToDesc(self, volname): volname = volname.strip() # Don't make these module global constants since we have to runtime # translate them anyway. monthdict = [ '', _('January'), _('February'), _('March'), _('April'), _('May'), _('June'), _('July'), _('August'), _('September'), _('October'), _('November'), _('December') ] for each in self._volre.keys(): match = re.match(self._volre[each], volname) # Let ValueErrors percolate up if match: year = int(match.group('year')) if each == 'quarter': d =["", _("First"), _("Second"), _("Third"), _("Fourth") ] ord = d[int(match.group('quarter'))] return _("%(ord)s quarter %(year)i") elif each == 'month': monthstr = match.group('month').lower() for i in range(1, 13): monthname = time.strftime("%B", (1999,i,1,0,0,0,0,1,0)) if monthstr.lower() == monthname.lower(): month = monthdict[i] return _("%(month)s %(year)i") raise ValueError, "%s is not a month!" % monthstr elif each == 'week': month = monthdict[int(match.group("month"))] day = int(match.group("day")) return _("The Week Of Monday %(day)i %(month)s %(year)i") elif each == 'day': month = monthdict[int(match.group("month"))] day = int(match.group("day")) return _("%(day)i %(month)s %(year)i") else: return match.group('year') raise ValueError, "%s is not a valid volname" % volname # The following two methods should be inverses of each other. -ddm def dateToVolName(self,date): datetuple=time.localtime(date) if self.ARCHIVE_PERIOD=='year': return time.strftime("%Y",datetuple) elif self.ARCHIVE_PERIOD=='quarter': if datetuple[1] in [1,2,3]: return time.strftime("%Yq1",datetuple) elif datetuple[1] in [4,5,6]: return time.strftime("%Yq2",datetuple) elif datetuple[1] in [7,8,9]: return time.strftime("%Yq3",datetuple) else: return time.strftime("%Yq4",datetuple) elif self.ARCHIVE_PERIOD == 'day': return time.strftime("%Y%m%d", datetuple) elif self.ARCHIVE_PERIOD == 'week': # Reconstruct "seconds since epoch", and subtract weekday # multiplied by the number of seconds in a day. monday = time.mktime(datetuple) - datetuple[6] * 24 * 60 * 60 # Build a new datetuple from this "seconds since epoch" value datetuple = time.localtime(monday) return time.strftime("Week-of-Mon-%Y%m%d", datetuple) # month. -ddm else: return time.strftime("%Y-%B",datetuple) def volNameToDate(self, volname): volname = volname.strip() for each in self._volre.keys(): match = re.match(self._volre[each],volname) if match: year = int(match.group('year')) month = 1 day = 1 if each == 'quarter': q = int(match.group('quarter')) month = (q * 3) - 2 elif each == 'month': monthstr = match.group('month').lower() m = [] for i in range(1,13): m.append( time.strftime("%B",(1999,i,1,0,0,0,0,1,0)).lower()) try: month = m.index(monthstr) + 1 except ValueError: pass elif each == 'week' or each == 'day': month = int(match.group("month")) day = int(match.group("day")) try: return time.mktime((year,month,1,0,0,0,0,1,-1)) except OverflowError: return 0.0 return 0.0 def sortarchives(self): def sf(a, b): al = self.volNameToDate(a) bl = self.volNameToDate(b) if al > bl: return 1 elif al < bl: return -1 else: return 0 if self.ARCHIVE_PERIOD in ('month','year','quarter'): self.archives.sort(sf) else: self.archives.sort() self.archives.reverse() def message(self, msg): if self.VERBOSE: f = sys.stderr f.write(msg) if msg[-1:] != '\n': f.write('\n') f.flush() def open_new_archive(self, archive, archivedir): index_html = os.path.join(archivedir, 'index.html') try: os.unlink(index_html) except: pass os.symlink(self.DEFAULTINDEX+'.html',index_html) def write_index_header(self): self.depth=0 print self.html_head() if not self.THREADLAZY and self.type=='Thread': self.message(_("Computing threaded index\n")) self.updateThreadedIndex() def write_index_footer(self): for i in range(self.depth): print '' print self.html_foot() def write_index_entry(self, article): subject = self.get_header("subject", article) author = self.get_header("author", article) if as_boolean(config.archiver.pipermail.obscure_email_addresses): try: author = re.sub('@', _(' at '), author) except UnicodeError: # Non-ASCII author contains '@' ... no valid email anyway pass subject = CGIescape(subject, self.lang) author = CGIescape(author, self.lang) d = { 'filename': urllib.quote(article.filename), 'subject': subject, 'sequence': article.sequence, 'author': author } print quick_maketext( 'archidxentry.html', d, mlist=self.maillist) def get_header(self, field, article): # if we have no decoded header, return the encoded one result = article.decoded.get(field) if result is None: return getattr(article, field) # otherwise, the decoded one will be Unicode return result def write_threadindex_entry(self, article, depth): if depth < 0: self.message('depth<0') depth = 0 if depth > self.THREADLEVELS: depth = self.THREADLEVELS if depth < self.depth: for i in range(self.depth-depth): print '' elif depth > self.depth: for i in range(depth-self.depth): print '