diff options
Diffstat (limited to 'contrib/mmdsr')
| -rw-r--r-- | contrib/mmdsr | 572 |
1 files changed, 0 insertions, 572 deletions
diff --git a/contrib/mmdsr b/contrib/mmdsr deleted file mode 100644 index 25c52e0f7..000000000 --- a/contrib/mmdsr +++ /dev/null @@ -1,572 +0,0 @@ -#!/bin/sh -############################################################################### -# mmdsr -- Mailman Daily Status Report (cron job, run at 23:59) -############################################################################### -# Copyright (c) 2005, Brad Knowles -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# -# The name of the author or other contributors may not be used -# to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -############################################################################### - -############################################################################### -# Version history -############################################################################### -# 0.0.1 Initial creation by Brad Knowles <brad@stop.mail-abuse.org> -# Created on: Tue Feb 15 04:01:20 PST 2005 -# -# 0.0.2 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Wed Feb 16 18:55:52 CET 2005 -# Feedback from Tom G. Christensen (tgc99): -# > The current UID grab command doesn't work on Solaris -# > (2.6 & 8 tested). -# > -# > I'd recommend this instead: -# > ps -o user -p $$|tail -1 -# > -# > This is tested and works on RH 6.2, RH 7.3, RHEL 2.1, -# > RHEL3, FC3, FreeBSD 4.9, Solaris 2.6, 8. -# -# 0.0.3 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Sun Mar 13 01:15:24 CET 2005 -# Noted errors from grep when trying to search a nonexistant -# file. Added "-s" option to compensate, and extra code to -# notify the admin if the log file doesn't exist. -# -# 0.0.4 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Sun Mar 13 01:58:37 CET 2005 -# Eliminate some extra cruft from the "fromusenet" logs. -# -# 0.0.5 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Wed Apr 13 00:54:42 CEST 2005 -# Eliminate a lot of extra cruft from the "error" & "vette" logs. -# Eliminate more cruft from the "fromusenet" logs. -# -# 0.0.6 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Thu Apr 14 03:07:08 CEST 2005 -# Eliminate even more cruft from the "fromusenet" logs. -# Eliminate more cruft from the "vette" logs. -# Eliminate cruft from the "mischief" logs. -# -# 0.0.7 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Sun May 1 22:29:13 CEST 2005 [guess] -# The "error" log "no such list" errors didn't need to be -# quite so "compressed", open them up to be more readable. -# Also eliminate all use of "xargs", use "fmt -w 75" instead. -# -# 0.0.8 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Sun May 1 23:09:13 CEST 2005 -# Clean up code for queue subdirectories to check. -# -# 0.0.9 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Wed May 11 01:23:40 CEST 2005 -# Eliminate date/time stamps from the rest of the logs, -# then pipe through `sort | uniq -c | sort -nr` -# so that all possible duplicates are eliminated. -# -# 0.0.10 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Tue Jul 12 16:53:49 CDT 2005 -# Add code to do summary of normal list activity -# -# 0.0.11 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Tue Sep 6 15:44:19 CEST 2005 -# Tweaked display of Mailman/qfiles subdirectories -# so as to avoid displaying more than $MAX_LINES -# output, which really helps if you have thousands -# of bounces sitting around, etc.... -# Also slightly tweaked display of fromusenet logs -# to split summary from errors. -# -# 0.0.12 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Thu Sep 22 22:37:35 CEST 2005 -# Bugs found by Mark Sapiro and Adrian Wells. -# Mark suggested splitting the Mailman log directory -# from the rest of the Mailman source and queues, -# as well as making sure to clean up all temp files. -# Adrian discovered that there was a log file format -# change between Mailman 2.1.5 and 2.1.6, which broke -# hourly statistics. -# -# 0.0.13 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Mon Dec 26 05:54:27 CET 2005 -# Bugs found by Tom G. Christensen (tgc99): -# > ps output on solaris is full of whitespace but a further -# > echo get's rid of it. -# > The lines in the smtp log are sometimes broken up by a -# > newline (right before the msgid) which throws of the -# > summary. Piping it through sed first will rejoin the broken -# > lines. -# > Use $AWK instead of awk. -# Enhancements from Mark Sapiro: -# > The vette log summary lists posts held for moderation -# > individually under "Other Errors:". The following patch -# > (watch out for wrapped lines) summarizes them by list instead. -# -# 0.0.14 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Thu Dec 29 08:17:38 CET 2005 -# Added code to check /usr/local/mailman/data for -# moderation hold queue -# -# 0.0.15 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Thu Jan 26 02:39:38 CET 2006 -# Tweaked display of summary data in the "smtp-failure" log, -# and "other" category in the "vette" log, so as to reduce -# the spewage when things go wonky, mostly by removing -# unique-ifying data like message-id or sender address. -# Also tweaked moderation queue information, to tell us -# how many Python pickle files are in the data directory, -# as opposed to just doing a directory listing and skipping -# the files in the middle if there are too many -- now we -# know how many are being skipped. -# -# 0.0.16 Update by Brad Knowles <brad@stop.mail-abuse.org> -# Updated on: Sun Jan 29 11:45:58 CET 2006 -# Mark found a couple of typos that I somehow let slip -# through. Thanks! - -############################################################################### -# Set up locations of standard commands, directories, etc.... -# You may need to modify these for your platform or installation. -############################################################################### - -AWK="/usr/bin/awk" -BASENAME="/usr/bin/basename" -CAT="/bin/cat" -DATE="/bin/date" -EGREP="/bin/egrep" -FMT="/usr/bin/fmt" -GREP="/bin/grep" -HEAD="/usr/bin/head" -LS="/bin/ls" -PS="/bin/ps" -RM="/bin/rm" -SED="/bin/sed" -SENDMAIL="/usr/sbin/sendmail" -SLEEP="/bin/sleep" -SORT="/usr/bin/sort" -TAIL="/usr/bin/tail" -TOUCH="/bin/touch" -TR="/usr/bin/tr" -UNIQ="/usr/bin/uniq" -XARGS="/usr/bin/xargs" -WC="/usr/bin/wc" - -############################################################################### -# Directory for temporary files -############################################################################### -TMPDIR="/tmp" - -############################################################################### -# Mailman queue directory -############################################################################### -QUEUEDIR="/usr/local/mailman/qfiles" - -############################################################################### -# Mailman log directory -############################################################################### -LOGDIR="/usr/local/mailman/logs" - -############################################################################### -# Mailman data directory -############################################################################### -DATADIR="/usr/local/mailman/data" - -############################################################################### -# Maximum number of subdirectory entries to display in report -############################################################################### -MAX_QUEUE_LINES=20 - -############################################################################### -# Maximum number of moderation queue pickle files to display in report -############################################################################### -MAX_DATA_LINES=100 - -############################################################################### -# Mailman Log files to check for errors. -# No need to specify path, only log file name. -############################################################################### -ERR_LOGS="error fromusenet locks mischief post qrunner smtp-failure vette" - -############################################################################### -# Mailman Log files to summarize. -# No need to specify path, only log file name. -############################################################################### -SUM_LOGS="fromusenet post smtp" - -############################################################################### -# Mailman queue subdirectories to check. No need to specify path, only name. -############################################################################### -SUBDIRS="archive bounces commands in news out retry shunt virgin" - -############################################################################### -# Specify recipients for report. If none, then simply print to "STDOUT". -# Specify sender address for report. Not used if recipient list is empty. -############################################################################### - -SENDER="INSERT.YOUR.SENDER.ADDRESS@HERE" -RCPTS="INSERT.YOUR.RECIPIENTS.ADDRESSES@HERE" - -############################################################################### -# If you run this program in cron at 23:59:00, you need to sleep sixty -# seconds to make sure that you capture all the logs for the previous day. -# Comment this line out to disable sleeping altogether. -############################################################################### - -SLEEPTIME=60 - -############################################################################### -# What user id is Mailman installed under? -# How do we determine what UID this program is running as? -# NB: There will probably be SysV vs. BSD semantic differences here. -# Be careful. Make sure the command you specify actually works. -# The command string specified here may seem arcane, but should pull -# out the information we need, and throw away everything else. If -# there is an easier cross-platform way to do it, please let me know. -############################################################################### - -GRABUID=`$PS -o user -p $$ | $TAIL -1` -MYUID=`echo $GRABUID` -RUNAS="mailman" - -############################################################################### -# No modifications below this line should be required. -############################################################################### - -DAY=`$DATE '+%b %d'` -YEAR=`$DATE '+%Y'` - -PROG=`$BASENAME $0` - -if [ "${MYUID}" != "${RUNAS}" -a "${MYUID}" != "root" ] ; then - echo "ERROR: You must be root or ${RUNAS} to run $PROG." - exit 1 -fi - -if [ "${SLEEPTIME}" != "" -a "${RCPTS}x" != "${SENDER}x" ] ; then - if [ $SLEEPTIME -gt 0 ] ; then - $SLEEP $SLEEPTIME - fi -fi - -TMP="$TMPDIR/.$PROG.$$" -TMPLOG="$TMP.log" -$RM -f $TMP -$TOUCH $TMP - -if [ "${RCPTS}x" != "x" ] ; then - echo "From: ${SENDER}" >> $TMP - echo "To: ${RCPTS}" >> $TMP - echo "Subject: Mailman Daily Status Report: $DAY $YEAR" >> $TMP - echo "" >> $TMP -fi - -echo " Mailman Daily Status Report: $DAY $YEAR" >> $TMP -echo "" >> $TMP -echo "" >> $TMP - -echo "******************************" >> $TMP -echo "Log File Summary" >> $TMP -echo "******************************" >> $TMP -echo "" >> $TMP - -for LOG in $SUM_LOGS -do - - $RM -f $TMPLOG - $TOUCH $TMPLOG - echo "Log file: $LOG" >> $TMP - echo "==============================" >> $TMP - - if [ -f "$LOGDIR/${LOG}" ] ; then - $SED -e :a -e '$!N;s/\n //;ta' -e 'P;D' $LOGDIR/$LOG | $GREP -si "^$DAY [0-9][0-9:]* $YEAR" >> $TMPLOG - - if [ "${LOG}" = "post" ] ; then - - echo "" >> $TMP - echo "Hourly Summary of Posts" >> $TMP - echo "-----------------------" >> $TMP - - $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.*$//' $TMPLOG | $UNIQ -c | $SORT -n +1 | $AWK '{ printf( "%8d %02d:00-%02d:59\n", $1, $2, $2 ) }' >> $TMP - - echo "" >> $TMP - echo "Post Count by List" >> $TMP - echo "------------------" >> $TMP - - $SED -e 's/^.* post to //' -e 's/ .*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Post Count by Sender" >> $TMP - echo "--------------------" >> $TMP - - $SED -e 's/^.* from //' -e 's/,.*$//' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "fromusenet" ] ; then - - GRPS=`$EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -i ' gating ' | $AWK '{ print $7 }' | $SORT -u | $FMT -w 75` - - for GRP in $GRPS - do - echo "" >> $TMP - echo "$GRP Article #'s Gated:" >> $TMP - echo "------------------------------" >> $TMP - $EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -i " gating " | $GREP -i " $GRP " | $SED -e 's/^.*\[//' -e 's/\]$//' -e 's/\.\./ /' | $TR ' ' '\n' | $SORT -u | $FMT -w 75 >> $TMP - done - - elif [ "${LOG}" = "smtp" ] ; then - - echo "" >> $TMP - echo "Hourly Summary of Messages Sent" >> $TMP - echo "-------------------------------" >> $TMP - $SED -e 's/^[A-Z][a-z][a-z] *[0-9]* //' -e 's/:.* for / /' -e 's/ recips,.*$//' $TMPLOG | $AWK '{ val=int($1); sum[val]+=$2 } END { for (i=0; i<24; i++) { printf "%8d %02d:00-%02d:59\n", sum[i], i, i } }' >> $TMP - - else - - $SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - fi - - else - - echo " ### Log file ${LOG} does not exist ### " >> $TMP - - fi - - echo "" >> $TMP - -done - -echo "******************************" >> $TMP -echo "Log File Squawks" >> $TMP -echo "******************************" >> $TMP -echo "" >> $TMP - -for LOG in $ERR_LOGS -do - $RM -f $TMPLOG - $TOUCH $TMPLOG - echo "Log file: $LOG" >> $TMP - echo "==============================" >> $TMP - $GREP -si "^$DAY [0-9][0-9:]* $YEAR" $LOGDIR/$LOG >> $TMPLOG - - if [ -f "$LOGDIR/${LOG}" ] ; then - - if [ "${LOG}" = "error" ] ; then - - echo "Uncaught Runner Exceptions:" >> $TMP - echo "------------------------------" >> $TMP - $GREP 'Uncaught runner exception' $TMPLOG | $SED 's/^.*exception: //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "No Such List:" >> $TMP - echo "------------------------------" >> $TMP - $GREP 'No such list' $TMPLOG | $SED -e 's/^.* "//' -e 's/".*$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - CNT=`$GREP -i 'shunting' $TMPLOG | $WC -l` - if [ "${CNT}x" != "x" ] ; then - if [ ${CNT} -gt 0 ] ; then - echo "" >> $TMP - echo "Shunting Count: $CNT" >> $TMP - echo "------------------------------" >> $TMP - fi - fi - - echo "" >> $TMP - echo "Other Errors:" >> $TMP - echo "------------------------------" >> $TMP - $EGREP -vi '(Uncaught runner exception|No such list|Ignoring unparseable message|Traceback|shunting)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "fromusenet" ] ; then - - $EGREP -vi '(watermark:|nothing new| posted to |: \[[0-9\.]*\])' $TMPLOG | $GREP -vi " gating " | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "mischief" ] ; then - - echo "" >> $TMP - echo "Login failure with private rosters (by user):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Login failure with private rosters' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Unsub attempt of non-member (by user):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Unsub attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Reminder attempt of non-member (by user):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Reminder attempt of non-member' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Other Messages:" >> $TMP - echo "------------------------------" >> $TMP - $EGREP -vi '(Login failure with private rosters|Unsub attempt of non-member|Reminder attempt of non-member)' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "post" ] ; then - - $GREP -vi 'success' $TMPLOG | $SED 's/^.* ([0-9]*) //' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "vette" ] ; then - - echo "" >> $TMP - echo "Message held -- Post by non-member (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP 'Post by non-member' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Message held -- Suspicious header (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'suspicious header' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Discarded posting (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Discarded posting' $TMPLOG | $AWK '{ print $6 }' | $SED 's/:$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Bulk/Junk message discarded (by list):" >> $TMP - echo "------------------------------" >> $TMP - $EGREP -i '(bulk|junk) message discarded' $TMPLOG | $AWK '{ print $NF }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Implicit destination (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Message has implicit destination' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Post to moderated newsgroup (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Posting to a moderated newsgroup' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Post to moderated list (by list):" >> $TMP - echo "------------------------------" >> $TMP - $GREP -i 'Post to moderated list' $TMPLOG | $AWK '{ print $6 }' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - echo "" >> $TMP - echo "Other Errors:" >> $TMP - echo "------------------------------" >> $TMP - $EGREP -vi '(Post by non-member|suspicious header|message approved|Discarded posting|bulk message discarded|junk message discarded|Message has implicit destination|Posting to a moderated newsgroup|Post to moderated list|Message discarded, msgid)' $TMPLOG | $SED -e 's/^.* ([0-9]*) //' -e 's/, message-id=<[^> ]*>:/:/' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - elif [ "${LOG}" = "smtp-failure" ] ; then - - $SED 's/^.* ([0-9]*) //' $TMPLOG | $SED 's/delivery to [^@ ]*@[^@ ]* failed with code/delivery failed with code/g' | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - else - - $SED 's/^.* ([0-9]*) //' $TMPLOG | $SORT | $UNIQ -c | $SORT -nr >> $TMP - - fi - else - - echo " ### Log file ${LOG} does not exist ### " >> $TMP - - fi - echo "" >> $TMP - -done - -echo "******************************" >> $TMP -echo "Queue Directory Contents" >> $TMP -echo "******************************" >> $TMP -echo "" >> $TMP - -cd $QUEUEDIR -for DIR in $SUBDIRS -do - - $RM -f $TMPLOG - $TOUCH $TMPLOG - $LS -la $DIR >> $TMPLOG - LINES=`$WC -l < $TMPLOG` - LINES=`expr $LINES - 3` - - echo "Subdirectory: $DIR" >> $TMP - echo "==============================" >> $TMP - echo "Entries: $LINES" >> $TMP - echo "------------------------------" >> $TMP - - if [ $LINES -le $MAX_QUEUE_LINES ] ; then - $CAT $TMPLOG >> $TMP - else - $HEAD $TMPLOG >> $TMP - echo "" >> $TMP - echo " More than $MAX_QUEUE_LINES total entries, skipping ..." >> $TMP - echo "" >> $TMP - $TAIL $TMPLOG >> $TMP - fi - echo "" >> $TMP - -done - -echo "******************************" >> $TMP -echo "Data Directory Contents" >> $TMP -echo " (Python pickle files only)" >> $TMP -echo "******************************" >> $TMP - -cd $DATADIR - -$RM -f $TMPLOG -$TOUCH $TMPLOG -$LS -la | $GREP -i '\.pck$' >> $TMPLOG -LINES=`$WC -l < $TMPLOG` - -echo "Python pickle files: $LINES" >> $TMP -echo "==============================" >> $TMP -echo "" >> $TMP - -if [ $LINES -le $MAX_DATA_LINES ] ; then - $CAT $TMPLOG >> $TMP -else - $HEAD $TMPLOG >> $TMP - echo "" >> $TMP - echo " More than $MAX_DATA_LINES total entries, skipping ..." >> $TMP - echo "" >> $TMP - $TAIL $TMPLOG >> $TMP -fi -echo "" >> $TMP - -echo "******************************" >> $TMP -echo "Moderation Queue Summary" >> $TMP -echo "******************************" >> $TMP -echo "" >> $TMP - -$AWK '{ print $NF }' $TMPLOG | $GREP -i '^heldmsg-' | $SED -e 's/^heldmsg-//' -e 's/-[0-9]*\.pck$//' | $SORT | $UNIQ -c | $SORT -nr >> $TMP -echo "" >> $TMP - -############################################################################### -# Okay, we're done with all the code stuff, now handle the output -############################################################################### - -if [ "${RCPTS}x" != "x" ] ; then - $SENDMAIL -t -f$SENDER < $TMP -else - $CAT $TMP -fi - -$RM $TMP $TMPLOG |