diff options
| -rwxr-xr-x | cron/gate_news | 117 |
1 files changed, 85 insertions, 32 deletions
diff --git a/cron/gate_news b/cron/gate_news index 32d928977..77f114e39 100755 --- a/cron/gate_news +++ b/cron/gate_news @@ -16,46 +16,99 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +"""Poll the NNTP servers for messages to be gatewayed to mailing lists. +""" + import os import marshal import paths +import nntplib +import errno from Mailman import MailList from Mailman import mm_cfg from Mailman import Utils +from Mailman import flock - -names = Utils.list_names() -try: - file = open(os.path.join(mm_cfg.DATA_DIR, "gate_watermarks"), "r") - watermarks = marshal.load(file) -except IOError, (x,y): - if x <> 2: - raise IOError, (x, y) - watermarks = {} - -for name in names: - if os.fork(): - continue - if watermarks.has_key(name): - wm = watermarks[name] - else: - wm = 0 - # Save the current state to .last, in case we crash while writing out, - # and corrupt the file. - list = MailList.MailList(name, lock=0) - file = open(os.path.join(mm_cfg.DATA_DIR, "gate_watermarks.last"), "w") - marshal.dump(watermarks, file) - file.close() - watermarks[name] = list.PollNewsGroup(wm) - # Save after every newsgroup... should probably save after every post. - # Reason being, want to make sure in case of a system crash or something, - # The same messages don't get gated multiple times. - file = open(os.path.join(mm_cfg.DATA_DIR, "gate_watermarks"), "w") - marshal.dump(watermarks, file) - file.close() - os._exit(0) - +WATERMARK_FILE = os.path.join(mm_cfg.DATA_DIR, 'gate_watermarks') +LIST_LOCK_FILE = os.path.join(mm_cfg.LOCK_DIR, 'gate_lock.') +def main(): + names = Utils.list_names() + try: + fp = open(WATERMARK_FILE) + watermarks = marshal.load(fp) + fp.close() + except IOError, (code, msg): + if code <> errno.ENOENT: + Utils.reraise() + watermarks = {} + # marshal or open could raise other exceptions, namely EOFError, + # ValueError or TypeError. TBD: should we zap the watermarks file if that + # happens? + for name in names: + # check to see if the list is gating news to mail. If not, skip the + # list. If so, then we have to poll the newsgroup and gate any + # outstanding messages. + mlist = MailList.MailList(name, lock=0) + if not mlist.gateway_to_mail: + continue + # try to get a per-list lock because it makes no sense to have more + # than one process gating a newsgroup. if we can't get the lock, just + # ignore the list for now... 5 minutes (usually how cron invokes this) + # later we'll try again anyway. We don't need to be anal about giving + # up the lock because we're setting a hung_time out of 4 minutes. + # This means that if we crashed, the next time the cron job runs, + # it'll just wax the lock and try again. + lock = flock.FileLock(LIST_LOCK_FILE + name, hung_timeout=240) + try: + lock.lock(timeout=0.001) + except flock.TimeOutError: + # someone else is gating this list already + continue + # open up a connection to the gated newsgroup. we want to get the + # watermark for the group in the parent process so that we can safely + # update the gate_watermarks file. we'll actually do the gating in a + # child process + conn = nntplib.NNTP(mlist.nntp_host) + r,c,first,last,n = conn.group(mlist.linked_newsgroup) + first = int(first) + last = int(last) + wm = watermarks.get(name, 0) + watermarks[name] = last + if wm <> 0: + # TBD: Essentially this does a mass catch-up on the newsgroup. + # The first time this script is run, no messages will be will be + # forwarded. We *could* have an option to control this, but who + # wants that? ;-) + if not os.fork(): + # in the child. + # + # steal the lock from the parent because we're going to manage + # it from here on, and we have a different PID than our + # parent. we't want to minimize any race conditions where + # someone else can steal the lock from us. I think there's + # still a race condition during the time we've actually got + # the file open for writing and when we're done writing it + # (during the steal()), but that should be very small. + lock.steal() + mlist.PollNewsGroup(conn, wm, first, last) + lock.unlock() + os._exit(0) + # Save the new watermarks after every newsgroup gating has + # started, so in case of a system crash we reduce the number of + # multiply gated messages. it might be better to save after every + # post, but that is harder to coordinate safely between the + # subprocesses, and would probably be *much* slower + omask = os.umask(002) + try: + fp = open(WATERMARK_FILE + '.tmp', 'w') + marshal.dump(watermarks, fp) + fp.close() + os.rename(WATERMARK_FILE + '.tmp', WATERMARK_FILE) + finally: + os.umask(omask) +if __name__ == '__main__': + main() |