#!/bin/bash

source /gpfs/wizard/flight/production/installed/set_pam_env.sh > /dev/null 2>&1;

if [ "$PAM_BIN" == "" ]; then exit; fi

export MAIL=Emiliano.Mocchiutti@ts.infn.it

#export CCMAIL=Emiliano.Mocchiutti@ts.infn.it
export CCMAIL=boezio@ts.infn.it

export PRODDIR=/gpfs/wizard/flight/production/

export WRKDIR=$PRODDIR/deamons/wrkdir/

export LOGDIR=$PRODDIR/logs/

MFILE=$WRKDIR/mailtime.ref

SENDMAIL=$WRKDIR/mail.log

MAXNTRY=4

LOG=$WRKDIR/log.log

RRBUSY=$WRKDIR/rrlist.busy

YODABUSY=$WRKDIR/yodalist.busy

DVBUSY=$WRKDIR/dvlist.busy

WAITFOR=10800

touch $LOG

touch $RRBUSY

touch $YODABUSY

touch $DVBUSY

touch $MFILE

touch $SENDMAIL

echo -n " DEAMON STARTED AT " >> $LOG;

date >> $LOG;

echo "600" > $WRKDIR/log.step 
#echo "-1" > $WRKDIR/log.step 

TIME=`tail -1 $WRKDIR/log.step`;

while [ $TIME -gt 0 ]; do

#
# RAWREADER OUTPUT
#
    ls $LOGDIR/rr.[0-9]*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	FILE=`basename $file`;
		
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
		    
	    PAMNAME0=`basename $file .log`
			
	    PAMNAME1=`echo $PAMNAME0 | sed s/rr.//g `.pam

	    RETRY=$LOGDIR/rr.$PAMNAME1.retry
		    
	    NTRY=0;

	    if [ -f $RETRY ]; then
			
		NTRY=`tail -1 $RETRY`;

		let NTRY=$NTRY+1;
			
	    fi;		    
		    
	    if [ $NTRY -gt $MAXNTRY ]; then
		    
		OKOUT=`grep -i "finished, exiting..." $file`

		if [ "$OKOUT" == "" ]; then

		    ERROR="no exiting signature";
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon RR: $ERROR in file $file" >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
			
		    echo "NTRY is $NTRY " >>$LOG;
		    
		    echo -n " File $file, error $ERROR move in bad dir and sending mail at " >>$LOG;

		    date >> $LOG;			

		    mv -f $file $LOGDIR/rr/bad/$FILE.NTRY;
			    
		else

		    ERROR="no way to process file $file";
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon RR: $ERROR " >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
			    
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " File $file, error $ERROR move in bad dir and sending mail at " >>$LOG;
		    
		    date >> $LOG;			

		    mv -f $file $LOGDIR/rr/bad/$FILE.NTRY;

		fi;

	    else

		OKOUT=`grep -i "finished, exiting..." $file`

		if [ "$OKOUT" == "" ]; then
			
		    echo $NTRY > $RETRY;
 
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " RR on $file aborted move in bad dir at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    mv -f $file $LOGDIR/rr/bad/$FILE.$NTRY;
			    
		    CHOK=`tail -1 $RRBUSY`;
						
		    if [ "$CHOK" == "" ]; then
							    
			echo "0" > $RRBUSY;
				    
			CHOK=`tail -1 $RRBUSY`;
				
		    fi
		    
		    while [ $CHOK -eq 1 ]; do
					    
			echo -n " RR LIST FILE IS BUSY " >> $LOG;
						
			date >> $LOG;
						
			sleep 1;
				
			CHOK=`tail -1 $RRBUSY `;
					
		    done
						
		    echo "1" > $RRBUSY;
			    
		    echo " Force resubmitting of job for file $PAMNAME1 " >>$LOG;
			    
		    echo -n " Rename entry $PAMNAME1 to ${PAMNAME1}_rrfailed in rr.rrlist at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    cat $WRKDIR/rr.rrlist | sed s/$PAMNAME1/${PAMNAME1}_rrfailed/g 1> $WRKDIR/rr.rrlist.tmp;
			    
		    cat $WRKDIR/rr.rrlist >> $WRKDIR/rr.rrlist.old;
		    
		    mv -f $WRKDIR/rr.rrlist.tmp $WRKDIR/rr.rrlist;
			    
		    echo "0" > $RRBUSY;

		else
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " File $file is good, moving log to good dir at " >>$LOG;

		    date >> $LOG;

		    if [ -f $RETRY ]; then
				
			mv -f $RETRY $LOGDIR/rr/good/;
					
		    fi;

		    while [ -f $LOGDIR/rr/good/$FILE.$NTRY ]; do

			let NTRY=$NTRY+1;

		    done;

		    mv -f $file $LOGDIR/rr/good/$FILE.$NTRY;
			    
###			    RNAME0=`basename $file .log`.pam
###			    RNAME=`echo $RNAME0 | sed 's/rr.//g'`			    
###			    rm -f $PRODDIR/preRawreader/$RNAME

		fi;
			
		if [ -f $file ] && [ $DELTA -gt 43200 ]; then
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " Warning! Very old RR job in $file send mail at " >>$LOG;
			    
		    date >> $LOG;						

		    ERROR="RR job in unknown status,";

		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon RR: $ERROR file $file" >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;

		fi;

	    fi;

	fi;
		
    done;

#
# RAWREADER LSF JOBS
#
    ls $LOGDIR/rr.lsf.*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	FILE=`basename $file`;
	
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
			
	    PAMNAME0=`basename $file .log`
			
	    PAMNAME1=`echo $PAMNAME0 | sed s/rr.lsf.//g `.pam

	    RETRY=$LOGDIR/rr.$PAMNAME1.retry
		    
	    NTRY=0;

	    if [ -f $RETRY ]; then
			
		NTRY=`tail -1 $RETRY`;

		let NTRY=$NTRY+1;
			
	    fi;		    
		    
	    if [ $NTRY -gt $MAXNTRY ]; then

		ERROR="no way to process file $file";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon RR: $ERROR " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

		echo "NTRY is $NTRY " >>$LOG;

		echo -n " File $file, error $ERROR move in bad dir and sending mail at " >>$LOG;

		date >> $LOG;			

		mv -f $file $LOGDIR/rr/bad/$FILE.$NTRY;
			
	    else
		    
		OKOUT=`grep -i "successfully completed" $file`

		if [ "$OKOUT" == "" ]; then

		    echo $NTRY > $RETRY;
 
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " LSF job $file aborted move in bad dir at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    mv -f $file $LOGDIR/rr/bad/$FILE.$NTRY;
			    
		    CHOK=`tail -1 $RRBUSY`;
						
		    if [ "$CHOK" == "" ]; then
							    
			echo "0" > $RRBUSY;
				    
			CHOK=`tail -1 $RRBUSY`;
				
		    fi
			    
		    while [ $CHOK -eq 1 ]; do
					    
			echo -n " RR LIST FILE IS BUSY " >> $LOG;
						
			date >> $LOG;
						
			sleep 1;
				
			CHOK=`tail -1 $RRBUSY `;
					
		    done
						
		    echo "1" > $RRBUSY;
			    
		    echo " Force resubmitting of job for file $PAMNAME1 " >>$LOG;
			    
		    echo -n " Rename entry $PAMNAME1 to ${PAMNAME1}_failed in rr.rrlist at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    cat $WRKDIR/rr.rrlist | sed s/$PAMNAME1/${PAMNAME1}_failed/g 1> $WRKDIR/rr.rrlist.tmp;
			    
		    cat $WRKDIR/rr.rrlist >> $WRKDIR/rr.rrlist.old;
			    
		    mv -f $WRKDIR/rr.rrlist.tmp $WRKDIR/rr.rrlist;
			    
		    echo "0" > $RRBUSY;
			    
		else
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " LSF job $file done move in good dir at " >>$LOG;
				
		    date >> $LOG;						
				
		    while [ -f $LOGDIR/rr/good/$FILE.$NTRY ]; do
				
			let NTRY=$NTRY+1;

		    done;

		    mv -f $file $LOGDIR/rr/good/$FILE.$NTRY;
				
		    if [ -f $RETRY ]; then
				
			mv -f $RETRY $LOGDIR/rr/good/;
					
		    fi;
			    
		fi;

	    fi;

	    if [ -f $file ] && [ $DELTA -gt 43200 ]; then
			
		echo "NTRY is $NTRY " >>$LOG;

		echo -n " Warning! Very old LSF job in $file send mail at " >>$LOG;
		    
		date >> $LOG;						

		ERROR="LSF job in unknown status,";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon RR: $ERROR file $file " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

	    fi;
			
	fi;
		
    done;

#
# YODA OUTPUT
#
    ls $LOGDIR/yoda.[0-9]*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	TIME=`tail -1 $WRKDIR/log.step `;
		    
	if [ $TIME -le 0 ]; then

	    echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

	    date >> $LOG;

	    exit;

	fi

	while [ $TIME -eq 1 ]; do

	    TIME=`tail -1 $WRKDIR/log.step `;
		    		    
	    echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

	    date >> $LOG;

	    sleep 120;

	done

	FILE=`basename $file`;
		
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
		    
	    PAMNAME0=`basename $file .log`
			
	    PAMNAME1=`echo $PAMNAME0 | sed s/yoda.//g `.pam

	    RETRY=$LOGDIR/yoda.${PAMNAME1}.retry
		    
	    NTRY=0;

	    if [ -f $RETRY ]; then
			
		NTRY=`tail -1 $RETRY`;

		let NTRY=$NTRY+1;
			
	    fi;		    
		    
	    if [ $NTRY -gt $MAXNTRY ]; then
	
		OKOUT=`grep -i "finished, exiting..." $file`

		if [ "$OKOUT" == "" ]; then

		    ERROR="no exiting signature";
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon YODA: $ERROR in file $file " >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
				
		    echo "NTRY is $NTRY " >>$LOG;
		    
		    echo -n " File $file, YODA error $ERROR move in bad dir and sending mail at " >>$LOG;

		    date >> $LOG;			

		    mv -f $file $LOGDIR/yoda/bad/${FILE}.$NTRY;
			    
		else

		    ERROR="no way to process file $file";
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon YODA: $ERROR " >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
			    
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " File $file, YODA error $ERROR move in bad dir and sending mail at " >>$LOG;
			    
		    date >> $LOG;			

		    mv -f $file $LOGDIR/yoda/bad/$FILE.NTRY;

		fi;

	    else

		OKOUT=`grep -i "finished, exiting..." $file`

		if [ "$OKOUT" == "" ]; then
			
		    echo $NTRY > $RETRY;
 
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " YODA on $file aborted move in bad dir at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    mv -f $file $LOGDIR/yoda/bad/$FILE.$NTRY;
			    
		    CHOK=`tail -1 $YODABUSY`;
						
		    if [ "$CHOK" == "" ]; then
							    
			echo "0" > $YODABUSY;
				    
			CHOK=`tail -1 $YODABUSY`;
				
		    fi
			    
		    while [ $CHOK -eq 1 ]; do
					    
			echo -n " YODA LIST FILE IS BUSY " >> $LOG;
						
			date >> $LOG;
						
			sleep 1;
				
			CHOK=`tail -1 $YODABUSY `;
					
		    done

		    echo "1" > $YODABUSY;
			    
		    echo " Force resubmitting of job for file $PAMNAME1 " >>$LOG;
			    
		    echo -n " Rename entry $PAMNAME1 to ${PAMNAME1}_yodafailed in yoda.yodalist at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    cat $WRKDIR/yoda.yodalist | sed s/$PAMNAME1/${PAMNAME1}_yodafailed/g 1> $WRKDIR/yoda.yodalist.tmp;
			    
		    cat $WRKDIR/yoda.yodalist >> $WRKDIR/yoda.yodalist.old;
			    
		    mv -f $WRKDIR/yoda.yodalist.tmp $WRKDIR/yoda.yodalist;
			    
		    echo "0" > $YODABUSY;

		else
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " YODA File $file is good, moving log to good dir at " >>$LOG;

		    date >> $LOG;

		    if [ -f $RETRY ]; then
				
			mv -f $RETRY $LOGDIR/yoda/good/;
					
		    fi;

		    while [ -f $LOGDIR/yoda/good/$FILE.$NTRY ]; do

			let NTRY=$NTRY+1;

		    done;

		    mv -f $file $LOGDIR/yoda/good/${FILE}.$NTRY;

		    NFILE=$PRODDIR/postRawreader/$PAMNAME1 ;
			    
#
# HERE WE REMOVE POSTRAWREADER FROM DISK
#

		    echo " REMOVING FILE $NFILE FROM DISK " >> $LOG 2>&1;

		    rm -f $NFILE >> $LOG 2>&1;
				
		    YFILE=$PRODDIR/level0/`basename $NFILE .pam`.root

		    echo " MOVING FILE $YFILE TO $PRODDIR/level0/good/ " >> $LOG 2>&1;

		    mv -f $PRODDIR/level0/`basename $NFILE .pam`.root $PRODDIR/level0/good/ >> $LOG 2>&1;

		    mv -f $PRODDIR/level0/`basename $NFILE .pam`Corrupted.dat $PRODDIR/level0/good/ >> $LOG 2>&1;

		fi;
			
		if [ -f $file ] && [ $DELTA -gt 43200 ]; then
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " Warning! Very old YODA job in $file send mail at " >>$LOG;
		    
		    date >> $LOG;						

		    ERROR="YODA job in unknown status,";

		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon YODA: $ERROR file $file" >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;

		fi;

	    fi;

	fi;
	
    done;

#
# YODA LSF JOBS
#
    ls $LOGDIR/yoda.lsf.*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	FILE=`basename $file`;
	
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
			
	    PAMNAME0=`basename $file .log`
		
	    PAMNAME1=`echo $PAMNAME0 | sed s/yoda.lsf.//g `.pam

	    RETRY=$LOGDIR/yoda.${PAMNAME1}.retry
		    
	    NTRY=0;

	    if [ -f $RETRY ]; then
			
		NTRY=`tail -1 $RETRY`;

		let NTRY=$NTRY+1;
			
	    fi;		    
		    
	    if [ $NTRY -gt $MAXNTRY ]; then

		ERROR="no way to process file $file";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon YODA: $ERROR " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

		echo "NTRY is $NTRY " >>$LOG;

		echo -n " YODA File $file, error $ERROR move in bad dir and sending mail at " >>$LOG;

		date >> $LOG;			

		mv -f $file $LOGDIR/yoda/bad/${FILE}.$NTRY;
			
	    else
		    
		OKOUT=`grep -i "successfully completed" $file`

		if [ "$OKOUT" == "" ]; then

		    echo $NTRY > $RETRY;
 
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " YODA LSF job $file aborted move in bad dir at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    mv -f $file $LOGDIR/yoda/bad/${FILE}.$NTRY;
			    
		    CHOK=`tail -1 $YODABUSY`;
						
		    if [ "$CHOK" == "" ]; then
							    
			    echo "0" > $YODABUSY;
				    
			CHOK=`tail -1 $YODABUSY`;
				
		    fi
			    
		    while [ $CHOK -eq 1 ]; do
					    
			echo -n " YODA LIST FILE IS BUSY " >> $LOG;
						
			date >> $LOG;
						
			sleep 1;
				
			CHOK=`tail -1 $YODABUSY `;
					
		    done
						
		    echo "1" > $YODABUSY;
			    
		    echo " Force resubmitting of job for file $PAMNAME1 " >>$LOG;
			    
		    echo -n " Rename entry $PAMNAME1 to ${PAMNAME1}_yfailed in yoda.yodalist at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    cat $WRKDIR/yoda.yodalist | sed s/$PAMNAME1/${PAMNAME1}_yfailed/g 1> $WRKDIR/yoda.yodalist.tmp;
			    
		    cat $WRKDIR/yoda.yodalist >> $WRKDIR/yoda.yodalist.old;
			    
		    mv -f $WRKDIR/yoda.yodalist.tmp $WRKDIR/yoda.yodalist;
			    
		    echo "0" > $YODABUSY;
			    
		else
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " YODA GRID job $file done move in good dir at " >>$LOG;
				
		    date >> $LOG;						
		
		    while [ -f $LOGDIR/yoda/good/$FILE.$NTRY ]; do
				
			let NTRY=$NTRY+1;

		    done;

		    mv -f $file $LOGDIR/yoda/good/${FILE}.$NTRY;
				
		    if [ -f $RETRY ]; then
				
			mv -f $RETRY $LOGDIR/yoda/good/;
				
		    fi;
			    
		fi;
			
	    fi;

	    if [ -f $file ] && [ $DELTA -gt 43200 ]; then
			
		echo "NTRY is $NTRY " >>$LOG;

		echo -n " Warning! Very old YODA LSF job in $file send mail at " >>$LOG;
			    
		date >> $LOG;						

		ERROR="LSF job in unknown status,";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon YODA: $ERROR file $file " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

	    fi;
			
	fi;

    done;

#
# YodaProfiler OUTPUT
#
    ls $LOGDIR/profiler.[0-9]*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	TIME=`tail -1 $WRKDIR/log.step `;
		    
	if [ $TIME -le 0 ]; then

	    echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

	    date >> $LOG;

	    exit;

	fi

	while [ $TIME -eq 1 ]; do

	    TIME=`tail -1 $WRKDIR/log.step `;
		    		    
	    echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

	    date >> $LOG;

	    sleep 120;

	done

	FILE=`basename $file`;
		
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
		    
	    PAMNAME0=`basename $file .log`
			
	    PAMNAME1=`echo $PAMNAME0 | sed s/profiler.//g `.pam

	    OKOUT=`grep -i "finished, exiting..." $file`

	    if [ "$OKOUT" == "" ]; then

		ERROR="no exiting signature";
			    
		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon YodaProfiler: $ERROR in file $file " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;
	    
		echo -n " File $file, YodaProfiler error $ERROR move in bad dir and sending mail at " >>$LOG;

		date >> $LOG;			

		mv -f $file $LOGDIR/profiler/bad/$FILE;
			    
	    else

		OKOUT=`grep -i "error (" $file`

		if [ "$OKOUT" != "" ]; then
			
		    ERROR=$OKOUT;
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon YodaProfiler: \"$ERROR\" in file $file " >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
		    
		    echo -n " File $file, YodaProfiler error \"$ERROR\" move in bad dir and sending mail at " >>$LOG;

		    date >> $LOG;			

		    mv -f $file $LOGDIR/profiler/bad/$FILE;

		else
			
		    echo -n " YodaProfiler File $file is good, moving log to good dir at " >>$LOG;

		    date >> $LOG;

		    mv -f $file $LOGDIR/profiler/good/$FILE;

		fi;

	    fi;

	fi;
		
    done;



#
# DARTHVADER LSF JOBS
#
    ls $LOGDIR/dv.lsf.*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	FILE=`basename $file`;
	
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
			
	    PAMNAME0=`basename $file .log`
		
	    PAMNAME1=`echo $PAMNAME0 | sed s/dv.lsf.//g `.pam

	    RETRY=$LOGDIR/dv.${PAMNAME1}.retry
		    
	    NTRY=0;

	    if [ -f $RETRY ]; then
			
		NTRY=`tail -1 $RETRY`;

		let NTRY=$NTRY+1;
			
	    fi;		    
		    
	    if [ $NTRY -gt $MAXNTRY ]; then

		ERROR="no way to process file $file";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon DarthVader: $ERROR " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

		echo "NTRY is $NTRY " >>$LOG;

		echo -n " DarthVader File $file, error $ERROR move in bad dir and sending mail at " >>$LOG;

		date >> $LOG;			

		mv -f $file $LOGDIR/dv/bad/${FILE}.$NTRY;
			
	    else
		    
		OKOUT=`grep -i "successfully completed" $file`

		if [ "$OKOUT" == "" ]; then

		    echo $NTRY > $RETRY;
 
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " DarthVader LSF job $file aborted move in bad dir at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    mv -f $file $LOGDIR/dv/bad/${FILE}.$NTRY;
			    
		    CHOK=`tail -1 $DVBUSY`;
						
		    if [ "$CHOK" == "" ]; then
							    
			    echo "0" > $DVBUSY;
				    
			CHOK=`tail -1 $DVBUSY`;
				
		    fi
			    
		    while [ $CHOK -eq 1 ]; do
					    
			echo -n " DV LIST FILE IS BUSY " >> $LOG;
						
			date >> $LOG;
						
			sleep 1;
				
			CHOK=`tail -1 $DVBUSY `;
					
		    done
						
		    echo "1" > $DVBUSY;
			    
		    echo " Force resubmitting of job for file $PAMNAME1 " >>$LOG;
			    
		    echo -n " Rename entry $PAMNAME1 to ${PAMNAME1}_yfailed in dv.dvlist at " >>$LOG;
			    
		    date >> $LOG;			
			    
		    cat $WRKDIR/dv.dvlist | sed s/$PAMNAME1/-${PAMNAME1}/g 1> $WRKDIR/dv.dvlist.tmp;
			    
		    cat $WRKDIR/dv.dvlist >> $WRKDIR/dv.dvlist.old;
			    
		    mv -f $WRKDIR/dv.dvlist.tmp $WRKDIR/dv.dvlist;
			    
		    echo "0" > $DVBUSY;
			    
		else
			
		    echo "NTRY is $NTRY " >>$LOG;

		    echo -n " DV GRID job $file done move in good dir at " >>$LOG;
				
		    date >> $LOG;						
		
		    while [ -f $LOGDIR/dv/good/$FILE.$NTRY ]; do
				
			let NTRY=$NTRY+1;

		    done;

		    mv -f $file $LOGDIR/dv/good/${FILE}.$NTRY;
				
		    if [ -f $RETRY ]; then
				
			mv -f $RETRY $LOGDIR/dv/good/;
				
		    fi;
			    
		fi;
			
	    fi;

	    if [ -f $file ] && [ $DELTA -gt 43200 ]; then
			
		echo "NTRY is $NTRY " >>$LOG;

		echo -n " Warning! Very old DarthVader LSF job in $file send mail at " >>$LOG;
			    
		date >> $LOG;						

		ERROR="LSF job in unknown status,";

		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon DarthVader: $ERROR file $file " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;

	    fi;
			
	fi;

    done;

#
# DarthVader OUTPUT
#
    ls $LOGDIR/dv.[0-9]*.log 1> $WRKDIR/log.list 2>/dev/null;

    for file in `cat $WRKDIR/log.list`; do

	TIME=`tail -1 $WRKDIR/log.step `;
		    
	if [ $TIME -le 0 ]; then

	    echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

	    date >> $LOG;

	    exit;

	fi

	while [ $TIME -eq 1 ]; do

	    TIME=`tail -1 $WRKDIR/log.step `;
		    		    
	    echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

	    date >> $LOG;

	    sleep 120;

	done

	FILE=`basename $file`;
		
	NOW=`date +%s`

	CTIME=`date --reference=$file +%s`

	let DELTA=NOW-CTIME

	if [ $DELTA -gt $WAITFOR ]; then 	
		    
	    PAMNAME0=`basename $file .log`
			
	    PAMNAME1=`echo $PAMNAME0 | sed s/dv.//g `.pam

	    OKOUT=`grep -i "finished, exiting..." $file`

	    if [ "$OKOUT" == "" ]; then

		ERROR="no exiting signature";
			    
		echo "################################################################################" >> $SENDMAIL;
		echo "Subject Logchecking deamon DarthVader: $ERROR in file $file " >> $SENDMAIL;
		echo "################################################################################" >> $SENDMAIL;
		cat $file >> $SENDMAIL;
	    
		echo -n " File $file, DarthVader error $ERROR move in bad dir and sending mail at " >>$LOG;

		date >> $LOG;			

		mv -f $file $LOGDIR/dv/bad/$FILE;
			    
	    else

		OKOUT=`grep -i "error (" $file`

		if [ "$OKOUT" != "" ]; then
			
		    ERROR=$OKOUT;
			    
		    echo "################################################################################" >> $SENDMAIL;
		    echo "Subject Logchecking deamon DarthVader: \"$ERROR\" in file $file " >> $SENDMAIL;
		    echo "################################################################################" >> $SENDMAIL;
		    cat $file >> $SENDMAIL;
		    
		    echo -n " File $file, DarthVader error \"$ERROR\" move in bad dir and sending mail at " >>$LOG;

		    date >> $LOG;			

		    mv -f $file $LOGDIR/dv/bad/$FILE;

		else
			
		    echo -n " DarthVader File $file is good, moving log to good dir at " >>$LOG;

		    date >> $LOG;

		    mv -f $file $LOGDIR/dv/good/$FILE;
		    
		fi;

	    fi;

	fi;
		
    done;

#
# HERE I HAVE TO MERGE L2 files and MOVE GOOD FILES IN THE GOOD DIRECTORY
#		    


#
# END, SEND MAIL WHEN IT IS THE CASE
#
    NOW=`date +%s`;

    CTIME=`date --reference=$MFILE +%s`;

    let DELTA=NOW-CTIME;

    if [ $DELTA -gt 43200 ]; then
			    
	touch $MFILE;
				
	if [ -s $SENDMAIL ]; then

	    TESTM=`grep -i subject $SENDMAIL`;

	    if [ "$TESTM" != "" ]; then
		
		echo -n " Sending report mail at " >> $LOG;
		
		date >> $LOG;

		cat $SENDMAIL | mail -s "Logchecking deamon report at `date`" -c $CCMAIL $MAIL
		    
	    fi;
		
	    rm -f $SENDMAIL;
		
	    touch $SENDMAIL;

	fi;

    fi;

    TIME=`tail -1 $WRKDIR/log.step `;
		
    if [ $TIME -le 0 ]; then

	echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

	date >> $LOG;

	exit;

    fi

    while [ $TIME -eq 1 ]; do

	TIME=`tail -1 $WRKDIR/log.step `;
		    		    
	echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

	date >> $LOG;

	sleep 120;

    done

    sleep $TIME;

    TIME=`tail -1 $WRKDIR/log.step `;

done;

echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

date >> $LOG;
