process_lists(): Rewritten for efficiency. The list object is never

locked until absolutely necessary.

1 files changed, 35 insertions, 30 deletions

diff --git a/cron/gate_news b/cron/gate_news
index dc8a95104..9f50337ab 100755
--- a/cron/gate_news
+++ b/cron/gate_news
@@ -81,11 +81,11 @@ def open_newsgroup(mlist):
 
 
 # This function requires the list to be locked.
-def poll_newsgroup(mlist, conn, first, last, lock):
+def poll_newsgroup(mlist, conn, first, last, glock):
     listname = mlist.internal_name()
     # NEWNEWS is not portable and has synchronization issues.
     for num in range(first, last):
-        lock.refresh()
+        glock.refresh()
         try:
             headers = conn.head(`num`)[3]
             found_to = 0
@@ -131,55 +131,60 @@ def poll_newsgroup(mlist, conn, first, last, lock):
 
 
 
-def process_lists(lock):
+def process_lists(glock):
     for listname in Utils.list_names():
-        lock.refresh()
+        glock.refresh()
         # Open the list unlocked just to check to see if it is gating news to
         # mail.  If not, we're done with the list.  Otherwise, lock the list
         # and gate the group.
         mlist = MailList.MailList(listname, lock=0)
         if not mlist.gateway_to_mail:
             continue
+        # Get the list's watermark, i.e. the last article number that we gated
+        # from news to mail.  `None' means that this list has never polled its
+        # newsgroup and that we should do a catch up.
+        watermark = getattr(mlist, 'usenet_watermark', None)
+        # Open the newsgroup, but let exceptions percolate up.
+        conn, first, last = open_newsgroup(mlist)
+        syslog('fromusenet', '%s: [%d..%d]' % (listname, first, last))
         try:
-            mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-        except LockFile.TimeOutError:
-            syslog('fromusenet', 'Could not acquire list lock: %s' %
-                   mlist.internal_name())
-            continue
-        try:
-            syslog('fromusenet', 'gating group %s to list %s' %
-                   (mlist.linked_newsgroup, listname))
-            # Get the list's watermark, i.e. the last article number that this
-            # gated from news to mail.  None means that this list has never
-            # polled its newsgroup.
-            watermark = getattr(mlist, 'usenet_watermark', None)
-            # Open the newsgroup, but let exceptions percolate up
-            conn, first, last = open_newsgroup(mlist)
             try:
                 if watermark is None:
-                    syslog('fromusenet',
-                           'catching up %s to article %d' % (listname, last))
+                    mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
                     # This is the first time we've tried to gate this
                     # newsgroup.  We essentially do a mass catch-up, otherwise
                     # we'd flood the mailing list.
                     mlist.usenet_watermark = last
+                    syslog('fromusenet', '%s caught up to article %d' %
+                           (listname, last))
                 else:
                     # The list has been polled previously, so now we simply
                     # grab all the messages on the newsgroup that have not
                     # been seen by the mailing list.  The first such article
-                    # is the maximum of the lowest article available on the
-                    # list and the watermark.  It's possible that some
+                    # is the maximum of the lowest article available in the
+                    # newsgroup and the watermark.  It's possible that some
                     # articles have been expired since the last time gate_news
                     # has run.  Not much we can do about that.
                     start = max(watermark+1, first)
-                    syslog('fromusenet', 'gating %s articles [%d..%d]' %
-                           (listname, start, last))
-                    poll_newsgroup(mlist, conn, start, last+1, lock)
-            finally:
-                conn.quit()
+                    if start > last:
+                        syslog('fromusenet', 'nothing new for list %s' %
+                               listname)
+                    else:
+                        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
+                        syslog('fromusenet', 'gating %s articles [%d..%d]' %
+                               (listname, start, last))
+                        # Use last+1 because poll_newsgroup() employes a for
+                        # loop over range, and this will not include the last
+                        # element in the list.
+                        poll_newsgroup(mlist, conn, start, last+1, glock)
+            except LockFile.TimeOutError:
+                syslog('fromusenet', 'Could not acquire list lock: %s' %
+                       listname)
         finally:
-            mlist.Save()
-            mlist.Unlock()
+            if mlist.Locked():
+                mlist.Save()
+                mlist.Unlock()
+            conn.quit()
         syslog('fromusenet', '%s watermark: %d' %
                (listname, mlist.usenet_watermark))
 
@@ -192,7 +197,7 @@ def main():
     try:
         lock.lock(timeout=0.5)
     except LockFile.TimeOutError:
-        syslog('fromusenet', 'could not acquire gate_news lock')
+        syslog('fromusenet', 'Could not acquire gate_news lock')
         return
     try:
         process_lists(lock)