diff options
Diffstat (limited to 'src/mailman/Archiver/HyperDatabase.py')
| -rw-r--r-- | src/mailman/Archiver/HyperDatabase.py | 339 |
1 files changed, 0 insertions, 339 deletions
diff --git a/src/mailman/Archiver/HyperDatabase.py b/src/mailman/Archiver/HyperDatabase.py deleted file mode 100644 index fecb544e8..000000000 --- a/src/mailman/Archiver/HyperDatabase.py +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -# -# site modules -# -import os -import marshal -import time -import errno - -# -# package/project modules -# -import pipermail -from flufl.lock import Lock, NotLockedError - -CACHESIZE = pipermail.CACHESIZE - -try: - import cPickle - pickle = cPickle -except ImportError: - import pickle - -# -# we're using a python dict in place of -# of bsddb.btree database. only defining -# the parts of the interface used by class HyperDatabase -# only one thing can access this at a time. -# -class DumbBTree: - """Stores pickles of Article objects - - This dictionary-like object stores pickles of all the Article - objects. The object itself is stored using marshal. It would be - much simpler, and probably faster, to store the actual objects in - the DumbBTree and pickle it. - - TBD: Also needs a more sensible name, like IteratableDictionary or - SortedDictionary. - """ - - def __init__(self, path): - self.current_index = 0 - self.path = path - self.lockfile = Lock(self.path + ".lock") - self.lock() - self.__dirty = 0 - self.dict = {} - self.sorted = [] - self.load() - - def __repr__(self): - return "DumbBTree(%s)" % self.path - - def __sort(self, dirty=None): - if self.__dirty == 1 or dirty: - self.sorted = self.dict.keys() - self.sorted.sort() - self.__dirty = 0 - - def lock(self): - self.lockfile.lock() - - def unlock(self): - try: - self.lockfile.unlock() - except NotLockedError: - pass - - def __delitem__(self, item): - # if first hasn't been called, we can skip the sort - if self.current_index == 0: - del self.dict[item] - self.__dirty = 1 - return - try: - ci = self.sorted[self.current_index] - except IndexError: - ci = None - if ci == item: - try: - ci = self.sorted[self.current_index + 1] - except IndexError: - ci = None - del self.dict[item] - self.__sort(dirty=1) - if ci is not None: - self.current_index = self.sorted.index(ci) - else: - self.current_index = self.current_index + 1 - - def clear(self): - # bulk clearing much faster than deleting each item, esp. with the - # implementation of __delitem__() above :( - self.dict = {} - - def first(self): - self.__sort() # guarantee that the list is sorted - if not self.sorted: - raise KeyError - else: - key = self.sorted[0] - self.current_index = 1 - return key, self.dict[key] - - def last(self): - if not self.sorted: - raise KeyError - else: - key = self.sorted[-1] - self.current_index = len(self.sorted) - 1 - return key, self.dict[key] - - def next(self): - try: - key = self.sorted[self.current_index] - except IndexError: - raise KeyError - self.current_index = self.current_index + 1 - return key, self.dict[key] - - def has_key(self, key): - return self.dict.has_key(key) - - def set_location(self, loc): - if not self.dict.has_key(loc): - raise KeyError - self.current_index = self.sorted.index(loc) - - def __getitem__(self, item): - return self.dict[item] - - def __setitem__(self, item, val): - # if first hasn't been called, then we don't need to worry - # about sorting again - if self.current_index == 0: - self.dict[item] = val - self.__dirty = 1 - return - try: - current_item = self.sorted[self.current_index] - except IndexError: - current_item = item - self.dict[item] = val - self.__sort(dirty=1) - self.current_index = self.sorted.index(current_item) - - def __len__(self): - return len(self.sorted) - - def load(self): - try: - fp = open(self.path) - try: - self.dict = marshal.load(fp) - finally: - fp.close() - except IOError, e: - if e.errno <> errno.ENOENT: raise - pass - except EOFError: - pass - else: - self.__sort(dirty=1) - - def close(self): - omask = os.umask(007) - try: - fp = open(self.path, 'w') - finally: - os.umask(omask) - fp.write(marshal.dumps(self.dict)) - fp.close() - self.unlock() - - -# this is lifted straight out of pipermail with -# the bsddb.btree replaced with above class. -# didn't use inheritance because of all the -# __internal stuff that needs to be here -scott -# -class HyperDatabase(pipermail.Database): - __super_addArticle = pipermail.Database.addArticle - - def __init__(self, basedir, mlist): - self.__cache = {} - self.__currentOpenArchive = None # The currently open indices - self._mlist = mlist - self.basedir = os.path.expanduser(basedir) - # Recently added articles, indexed only by message ID - self.changed={} - - def firstdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - datekey, msgid = self.dateIndex.first() - date = time.asctime(time.localtime(float(datekey[0]))) - except KeyError: - pass - return date - - def lastdate(self, archive): - self.__openIndices(archive) - date = 'None' - try: - datekey, msgid = self.dateIndex.last() - date = time.asctime(time.localtime(float(datekey[0]))) - except KeyError: - pass - return date - - def numArticles(self, archive): - self.__openIndices(archive) - return len(self.dateIndex) - - def addArticle(self, archive, article, subject=None, author=None, - date=None): - self.__openIndices(archive) - self.__super_addArticle(archive, article, subject, author, date) - - def __openIndices(self, archive): - if self.__currentOpenArchive == archive: - return - self.__closeIndices() - arcdir = os.path.join(self.basedir, 'database') - omask = os.umask(0) - try: - try: - os.mkdir(arcdir, 02770) - except OSError, e: - if e.errno <> errno.EEXIST: raise - finally: - os.umask(omask) - for i in ('date', 'author', 'subject', 'article', 'thread'): - t = DumbBTree(os.path.join(arcdir, archive + '-' + i)) - setattr(self, i + 'Index', t) - self.__currentOpenArchive = archive - - def __closeIndices(self): - for i in ('date', 'author', 'subject', 'thread', 'article'): - attr = i + 'Index' - if hasattr(self, attr): - index = getattr(self, attr) - if i == 'article': - if not hasattr(self, 'archive_length'): - self.archive_length = {} - l = len(index) - self.archive_length[self.__currentOpenArchive] = l - index.close() - delattr(self, attr) - self.__currentOpenArchive = None - - def close(self): - self.__closeIndices() - - def hasArticle(self, archive, msgid): - self.__openIndices(archive) - return self.articleIndex.has_key(msgid) - - def setThreadKey(self, archive, key, msgid): - self.__openIndices(archive) - self.threadIndex[key]=msgid - - def getArticle(self, archive, msgid): - self.__openIndices(archive) - if not self.__cache.has_key(msgid): - # get the pickled object out of the DumbBTree - buf = self.articleIndex[msgid] - article = self.__cache[msgid] = pickle.loads(buf) - # For upgrading older archives - article.setListIfUnset(self._mlist) - else: - article = self.__cache[msgid] - return article - - def first(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index + 'Index') - try: - key, msgid = index.first() - return msgid - except KeyError: - return None - - def next(self, archive, index): - self.__openIndices(archive) - index = getattr(self, index + 'Index') - try: - key, msgid = index.next() - return msgid - except KeyError: - return None - - def getOldestArticle(self, archive, subject): - self.__openIndices(archive) - subject = subject.lower() - try: - key, tempid=self.subjectIndex.set_location(subject) - self.subjectIndex.next() - [subject2, date]= key.split('\0') - if subject!=subject2: return None - return tempid - except KeyError: - return None - - def newArchive(self, archive): - pass - - def clearIndex(self, archive, index): - self.__openIndices(archive) - if hasattr(self.threadIndex, 'clear'): - self.threadIndex.clear() - return - finished=0 - try: - key, msgid=self.threadIndex.first() - except KeyError: finished=1 - while not finished: - del self.threadIndex[key] - try: - key, msgid=self.threadIndex.next() - except KeyError: finished=1 |